Python Crawlers

Tuesday, May 3, 2016

Java - Calculate Average With Array Input

// By Rami Jaloudi
// Java - Calculate Average With Array Input

import java.util.*;

public class CalculateAverageInputArray {
public static void main(String[] args) {

int Iterations;

System.out.print("Please enter the number of iterations to simulate: ");

Scanner scan = new Scanner(System.in);

Iterations = scan.nextInt();

double[] numbers = new double[Iterations];

System.out.print("Please enter the number of iterations to simulate "
+ Iterations + " times: " + "\n");

Scanner scan2 = new Scanner(System.in);
for(int i=0; i < Iterations ; i++)
numbers[i] = scan2.nextDouble();

//alternative methods: if you are looking to pre-define the array & with integers instead of double.
//define an array
//int[] numbers = new int[]{10,20,15,25,16,60,100};

/*
* Average value of array elements would be
* sum of all elements/total number of elements
*/

//calculate sum of all array elements
double sum=0;

for(int i=0; i < Iterations ; i++)
sum = sum + numbers[i];

//calculate average value
double average = sum / Iterations;

System.out.println("Average value of array elements is : " + average);
}
}

Sunday, March 20, 2016

Insertion Sort | Java

// Java
// Insertion Sort
// Sort.java
// By Rami Jaloudi, Programmer

import java.util.*;
public class Sort {

public static void main (String [] args){
int n = 0;
boolean run = true;
while(run){
Scanner in = new Scanner(System.in);
Scanner options = new Scanner(System.in);
System.out.println("Which option would you like to select?");
System.out.println(
"\n" +
"(1)Insertion Sort - Uses Random Integers based on your input of iterations 'n' "
+ "\n" +
"(2)Insertion Sort - Best Case Scenario; (Based on your input for how many integers we will populate, it will have an array that is in Asecending Order) or "
+ "\n" +
"(3)Insertion Sort - Worst Case Scenario; (Based on your input for how many integers we will populate, it will have an array that is in Descending Order)"
+ "\n" + "\n" +
"----------------------------------------------------------------------------------------------------------"
+ "\n"
+ "PLEASE NOTE THAT THIS PROGRAM RUNS REPITISOUSLY. YOU MAY HAVE TO SCROLL UP TO SEE YOUR PREVIOUS RESULTS."
+ "\n" +
"----------------------------------------------------------------------------------------------------------"
);

int optNumber;
optNumber = options.nextInt();

if(optNumber == 1){ //Insertion Sort
System.out.println("Enter the number of integers you would like to populate: [PROGRAM RE-RUNS AUTOMATICALLY]");
n = in.nextInt();
int A[] = new int[n];

populateArray(A);
System.out.println("-----------------------------------------");
System.out.println("UNSORTED ARRAY ");
printArray(A);

// sort the array
//insertionSort(A);
//System.out.println("\nAfter Sorting: ");
insertionSort(A);
System.out.println("-----------------------------------------");
printArray(A);

System.out.println("\n");

} else {
}

if(optNumber == 2){ // Insertion Sort - Best Case Scenario
System.out.println("Enter the number of integers you would like to populate: [PROGRAM RE-RUNS AUTOMATICALLY]");
n = in.nextInt();
int A[] = new int[n];

populateArray2(A);
System.out.println("Before Sorting: ");
System.out.println("-----------------------------------------");
printArray(A);

// sort the array
//insertionSort(A);
System.out.println("\nAfter Sorting: ");
insertionSort(A);
printArray(A);
System.out.println("-----------------------------------------");

System.out.println("\n");

} else {
}

if(optNumber == 3){ // Insertion Sort - Worst Case Scenario
System.out.println("Enter the number of integers you would like to populate: [PROGRAM RE-RUNS AUTOMATICALLY]");
n = in.nextInt();
int A[] = new int[n];

populateArray3(A);
System.out.println("Before Sorting: ");
System.out.println("-----------------------------------------");
printArray(A);

// sort the array
//insertionSort(A);
System.out.println("\nAfter Sorting: ");
insertionSort(A);
printArray(A);
System.out.println("-----------------------------------------");

System.out.println("\n");

} else {
}
}
}

// Insertion sort:
public static void insertionSort(int[] list) {
int comps = 0, COMPCOUNT = 0; int steps=0;

for(int i = 1; i < list.length; i++) { //3n = 2
int j = i; // 1
comps++;
steps++;

// comparing i with the sorted elements and then inserting it
// for e.g., sorted elements, [0..i-1]
while (j > 0 && list[j] < list[j - 1]) {
int temp = list[j];
steps++; // Counting Actual Steps (Not COMPCOUNT)
list[j] = list[j - 1]; //1
steps++; // Counting Actual Steps (Not COMPCOUNT)
list[j - 1] = temp; //1
steps++; // Counting Actual Steps (Not COMPCOUNT)
COMPCOUNT++;
j--;
steps++; // Counting Actual Steps (Not COMPCOUNT)
}
steps++; // Counting Actual Steps (Not COMPCOUNT)
//comps++; // counting when the condition is false
//COMPCOUNT++;
}
int n2 = comps * comps;
int n3 = (n2 - comps)/4;
int dev = (COMPCOUNT - n3);
int stepDev = steps - n2;

System.out.println(
"INSERTION SORT METHOD RESULTS: " + "\n"

+ "Iterations('n'):" + "\t" + "\t" + comps + "\n"
+ "-----------------------------------------" + "\n"
+ "(n^2):" + "\t" + "\t" + "\t" + "\t" + n2 + "\n"
+ "-----------------------------------------" + "\n"
+ "(n2 - n)/4 " + "\n" + "Key Comps Formula:"
+ "\t" + "\t" + n3 + "\n"
+ "-----------------------------------------" + "\n"
+ "COMPCOUNT:" + "\t" + "\t" + "\t" + COMPCOUNT + "\n"
+ "-----------------------------------------" + "\n"
+ "Difference between" + "\n"
+ "COMPCOUNT & (n2 - comps)/4:" + "\t" + dev + "\n"
+ "-----------------------------------------" + "\n"
+ "Actual Steps:" + "\t" + "\t" + "\t" + steps + "\n"
+ "-----------------------------------------" + "\n"
+ "Difference between" + "\n"
+ "Actual Steps & n^2" + "\n" + "from n^2:" + "\t" + "\t" + "\t" + stepDev + "\n"
+ "-----------------------------------------" + "\n");

}

public static void printArray(int[] B) {

System.out.println(Arrays.toString(B));
System.out.println("-----------------------------------------");
}

public static void populateArray(int[] B) {
for (int i = 0; i < B.length; i++) {
B[i] = (int) (Math.random() * 10000);
}
}

public static void populateArray2(int[] B) { // For Insertion Sort - Best Case Scenario
for (int i = 0; i < B.length; i++) {
B[i] = i + 1;
}
}

public static void populateArray3(int[] B) { // For Insertion Sort - Worst Case Scenario
for (int i = 0; i<B.length; i++) {
B[i] = B.length - i;
}
}
}

Friday, March 18, 2016

Java - Insertion Sort v. Bubble Sort | Algorothims

// Java
// Insertion Sort
// Sort.java
// Java - Insertion Sort v. Bubble Sort
// By Rami Jaloudi, Programmer

import java.util.*;

public class Sort {

public static void main (String [] args){

int n = 0;

boolean run = true;

while(run){

Scanner in = new Scanner(System.in);

System.out.println("Enter the number of integers you would like to randomly populate: [RUN AGAIN & COMPARE]");

n = in.nextInt();

int A[] = new int[n];

Scanner options = new Scanner(System.in);

System.out.println("Which option would you like to select?");

System.out.println("(1)Insertion Sort; "

+ "\n"

+ "(2)Bubble Sort; or "

+ "\n" +

"(3)Bubble Sort 2nd Version");

int optNumber;

optNumber = options.nextInt();

if(optNumber == 1){

populateArray(A);

System.out.println("Before Sorting: ");

System.out.println("-----------------------------------------");

printArray(A);

// sort the array

//insertionSort(A);

System.out.println("\nAfter Sorting: ");

insertionSort(A);

System.out.println("-----------------------------------------");

printArray(A);

System.out.println("\n");

} else {

}

if(optNumber == 2){

populateArray(A);

System.out.println("Before Sorting: ");

System.out.println("-----------------------------------------");

printArray(A);

// sort the array

//insertionSort(A);

System.out.println("\nAfter Sorting: ");

bubbleSort(A);

printArray(A);

System.out.println("-----------------------------------------");

System.out.println("\n");

} else {

}

if(optNumber == 3){

populateArray(A);

System.out.println("Before Sorting: ");

System.out.println("-----------------------------------------");

printArray(A);

// sort the array

//insertionSort(A);

System.out.println("\nAfter Sorting: ");

bubbleSort2(A);

printArray(A);

System.out.println("-----------------------------------------");

System.out.println("\n");

} else {

}

// Bubble sort:

public static void bubbleSort(int[] list) {

int comps = 0;

int COMPCOUNT = 0;

for(int j = 0; j < list.length - 1; j++) {

for(int i = 0; i < list.length - 1; i++) {

// count the number of comparisons

comps++;

// compare neighboring elements

if(list[i] > list[i+1]) {

// swap i with i+1

int temp = list[i];

list[i] = list[i + 1];

list[i + 1] = temp;

// count number of swaps

COMPCOUNT++;

}

//System.out.println(list);

System.out.println("BUBBLE SORT 1: " + "Comparisons: " + comps + " COMPCOUNT: " + COMPCOUNT);

}

//Variation of Bubble Sort (improved)

//Stop sorting when the array has become sorted & do not compare elements of the array with elements that are in their correct sorted position

public static void bubbleSort2(int[] list) {

int comps = 0;

int COMPCOUNT = 0;

boolean didSwap = true;

// didSwap tracks whether any elements of the array

// were swapped during the last iteration of the loop.

// If no elements were swapped, the the array is

// sorted, so we can stop.

for(int j = 0; j < list.length - 1 && didSwap; j++) {

didSwap = false;

// at this point, j elements are already in their

// correct sorted position, so we do not compare

// with the last j elements in the list

for(int i = 0; i < list.length - 1 - j; i++) {

// count the number of comparisons

comps++;

// compare neighboring elements

if(list[i] > list[i+1]) {

didSwap = true;

// swap i with i+1

int temp = list[i];

list[i] = list[i + 1];

list[i + 1] = temp;

// count number of swaps

COMPCOUNT++;

}

//System.out.println(list);

System.out.println("BUBBLE SORT 2: " + "Comparisons: " + comps + " COMPCOUNT: " + COMPCOUNT);

}

/**

* Insertion sort:

* - pick an element and put it into its correct

* position in the sorted part of the list

* - repeat list.length times

public static void insertionSort(int[] list) {

int comps = 0, COMPCOUNT = 0;

for(int i = 1; i < list.length; i++) {

int j = i;

// compare i with sorted elements and insert it

// sorted elements: [0..i-1]

while (j > 0 && list[j] < list[j - 1]) {

int temp = list[j];

list[j] = list[j - 1];

list[j - 1] = temp;

COMPCOUNT++;

comps++; // loop condition true

j--;

}

comps++; // checking loop condition when false

}

//System.out.println(list);

System.out.println("INSERTION SORT: " + "Comparisons: " + comps + " COMPCOUNT: " + COMPCOUNT);

}

public static void printArray(int[] B) {

System.out.println(Arrays.toString(B));

}

public static void populateArray(int[] B) {

for (int i = 0; i < B.length; i++) {

B[i] = (int) (Math.random() * 100);

}

Tuesday, September 15, 2015

Depth & Scrape Website for HTML Code & Create csv file containing Links

Depth & Scrape Website for HTML Code & Create csv file containing Links #Scrape Website for HTML Code and Create csv file containing Links
# Using Python 2.7.9 - By Rami Jaloudi, Programmer
import urllib
import re
import os

def crawl(site, depth, linksfile):
pattern = re.compile(r'href="(http://.*?)"')
f = open(linksfile, 'a+')
try:
    if depth < MAX_DEPTH:
      print 'crawling [%s]...' % site,
      print >> f, '[%s]' % site
      url = urllib.urlopen(site)
      content = url.read()
      print content
      hits = pattern.findall(content)
      for hit in hits:
        print >> f, hit
      print 'done.'
      print >> f, ''
      for hit in hits:
        crawl(hit, depth + 1, linksfile)

except:
    pass
f.close()

MAX_DEPTH=3
base = r'http://nytimes.com'
linksfile = r'links.txt'

if os.path.isfile(linksfile):
os.remove(linksfile)
crawl(base, 0, linksfile)

Python Crawlers | Scraping Links from Websites using Python 2.7.9

Depth & Scrape Website for HTML Code & Create csv file containing Links #Python 2.7.9
# By Rami Jaloudi, Programmer
#Scraping links with Python 2.7.9

import urllib
from bs4 import BeautifulSoup
import urlparse
import mechanize

url = "http://........com" # Enter your preferred URL
#url = "http://nytimes.com"
br = mechanize.Browser()
urls = [url]
visited = [url]
while len(urls)>0:
    try:
        br.open(urls[0])
        urls.pop(0)
        with open("links.csv", "a") as file:
            for link in br.links():
                newurl = urlparse.urljoin(link.base_url,link.url)
                b1 = urlparse.urlparse(newurl).hostname
                b2 = urlparse.urlparse(newurl).path
                newurl = "http://"+b1+b2
                #file.write(newurl + "\n")

                if newurl not in visited and urlparse.urlparse(url).hostname in newurl:
                    urls.append(newurl)
                    visited.append(newurl)
                    print newurl
                    # SaveFile = open("links.csv", "tw")
                    file.write(newurl + "\n\n")
                    # SaveFile.close()

    except:
        print "error"
        urls.pop(0)

'''
    print(the_page)
    SaveFile = open('the_page.txt', 'w')
    SaveFile.write(str(the_page))
    SaveFile.close()
'''

Thursday, June 11, 2015

Web Crawling - Using regex module (Python)

Depth & Scrape Website for HTML Code & Create csv file containing Links # This script uses the regex and urllib modules to crawl a list of sites for title tags.
# It simply provides the title tags, i.e., titles you would see on web browser tabs.
# By Rami Jaloudi, Programmer
import re, urllib
try:
    import urllib.request # this is if you are running on Python 3
except:
    pass

sites = 'cnn nytimes bloomberg'.split()

pat = re.compile(r'<title>.+?</title>+', re.I|re.M)

for s in sites:
    print('Searching:' + s)
    try:
        u = urllib.urlopen('http://' + s + '.com')
    except:
        u = urllib.request.urlopen('http://' + s + '.com')
    text = u.read()
    title = re.findall(pat, str(text))
    print title
    print '\n'

Web Crawling - Using urllib module (Python)

Depth & Scrape Website for HTML Code & Create csv file containing Links import urllib

# Web Crawling - Using urllib module (Python) # By Rami Jaloudi, Programmer # This script runs on Python 2.7
# If using Python 3, please note that the urllib module has been split into parts and renamed as follows:
# urllib.request, urllib.parse, and urllib.error

url = "http://www.________.com" # Need to enter URL

request = urllib.urlopen(url)

response = request.read()

print response