import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

return filter_out_nonduplicates('./screens/canny/')[0]

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
return cluster([[1,1], [1,1], [1,0], [1,3], [10,12], [10,11], [10,10], [20,20], [22,20], [21,21]])

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/canny/'
full_list = hamming_a_directory(directory)
panels = os.listdir(directory)
grid = nxn_grid_from_itertools_combo(panels, full_list)

return give_names(cluster(grid), panels, grid)

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/canny/'
full_list = hamming_a_directory(directory)
panels = os.listdir(directory)
grid = nxn_grid_from_itertools_combo(panels, full_list)

return give_names(cluster(grid, 3), panels, grid)

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/canny/'
ham = filter_out_nonduplicates(directory)
return give_names(cluster(ham[2]), ham[0], ham[2])

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/phash/'
ham = filter_out_nonduplicates(directory, False)
return give_names(cluster(ham[2]), ham[0], ham[2])

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")
import matplotlib.pyplot as plt

directory = './screens/phash/'
ham = filter_out_nonduplicates(directory, False)
return give_names(cluster(ham[2], 4, 6, 100), ham[0], ham[2])

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

import matplotlib.pyplot as plt

directory = './screens/phash/'
ham = filter_out_nonduplicates(directory, False)
clust = cluster(ham[2], 1, -1, 100, 1)[0]
plt.plot(range(1, len(clust)+1), clust)
plt.title('SSE vs K - Image Hashes')
plt.xlabel('Value of K')
plt.ylabel('SSE')
plt.show()

directory = './screens/canny/'
full_list = hamming_a_directory(directory)
panels = os.listdir(directory)
grid = nxn_grid_from_itertools_combo(panels, full_list)
clust = cluster(grid, 1, -1, 100, 1)[0]
plt.plot(range(1, len(clust)+1), clust)
plt.title('SSE vs K - Edge Detected Images')
plt.xlabel('Value of K')
plt.ylabel('SSE')
plt.show()

<<clustering>>
<<filter_dupes>>

from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np

directory = './screens/phash/'
resize_flag = False
panels = []
values = []
for image in os.listdir(directory):
    panels.append(image)
    val = list(Image.open(directory+image).getdata())
    values.append(val)

clst = cluster(values, 4)

return give_names(clst, panels, values)

from sklearn.decomposition import PCA

<<filter_dupes>>
<<clustering>>

import matplotlib.pyplot as plt
import numpy as np

directory = './screens/phash/'
resize_flag = False
panels = os.listdir(directory)
grid = nxn_grid_from_itertools_combo(panels, hamming_a_directory(directory, resize_flag))
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

clust = cluster(principalComponents)
names = give_names(clust, panels, principalComponents)

return names

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

directory = './screens/phash/'
resize_flag = False
panels = os.listdir(directory)
grid = nxn_grid_from_itertools_combo(panels, hamming_a_directory(directory, resize_flag))
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

clust = cluster(principalComponents)
names = give_names(clust, panels, principalComponents)


fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

marks = ['r', 'b', 'g']

for i, cluster in enumerate(clust):
    for pt in cluster:
	ax.scatter(pt[0], pt[1], pt[2], c=marks[i])

plt.title('10-img set with PCA')
plt.show()

<<filter_dupes>>
<<clustering>>
import matplotlib.pyplot as plt
import numpy as np
from sklearn.decomposition import PCA

directory = './screens/phashbig/'
resize_flag = False
panels = []
for image in os.listdir(directory): panels.append(image)
grid = nxn_grid_from_itertools_combo(panels, hamming_a_directory(directory, resize_flag))
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

clust = cluster(principalComponents, 2, 25, 100)

names = give_names(clust, panels, principalComponents)

for i, x in enumerate(names):
    path = "./screens/imgbig/clustered/" + str(i) + "/"
    for y in x: move_directory("./screens/imgbig/", path, y)

<<filter_dupes>>
<<clustering>>
import matplotlib.pyplot as plt
import numpy as np
from sklearn.decomposition import PCA

directory = './screens/cannybig/'
panels = []
for image in os.listdir(directory): panels.append(image)
grid = nxn_grid_from_itertools_combo(panels, hamming_a_directory(directory))
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

clust = cluster(principalComponents, 3, 25, 100)

names = give_names(clust, panels, principalComponents)

for i, x in enumerate(names):
    path = "./screens/imgbig/clustered/" + str(i) + "/"
    for y in x: move_directory("./screens/imgbig/", path, y)

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

import matplotlib.pyplot as plt
import numpy as np
from sklearn.decomposition import PCA

directory = './screens/phashbig/'
panels = []
for image in os.listdir(directory): panels.append(image)
grid = nxn_grid_from_itertools_combo(panels, hamming_a_directory(directory))
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

clust = cluster(principalComponents, 1, -1, 100, 1)[0]
plt.plot(range(1, len(clust)+1), clust)
plt.title('SSE vs K - Image Hashes using PCA')
plt.xlabel('Value of K')
plt.ylabel('SSE')
plt.show()

import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized

dire = './screens/canny/'
dest = './screens/cannyfilter/'

for image in os.listdir(dire):
    img_orig = Image.open(dire+image)
    img_new = img_orig.resize((100, 100))
    dir_save = dest+image
    if os.path.isdir(dest):
	img_new.save(dir_save)
    else:
	os.mkdir(dest)
	img_new.save(dir_save)

import numpy as np
import os
import pickle
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/cannyfilter/' 
# find distances for canny images
panels = []
for image in os.listdir(directory): panels.append(image)
ham = hamming_a_directory(directory, False)

picklecan = open("cannyhamming.~ATH", 'wb')
pickle.dump(ham, picklecan)
picklecan.close()

import numpy as np
import os
import pickle
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/phash/' 
# find distances for canny images
panels = []
for image in os.listdir(directory): panels.append(image)
ham = hamming_a_directory(directory, False)

picklecan = open("hashhamming.~ATH", 'wb')
pickle.dump(ham, picklecan)
picklecan.close()

import cPickle as pickle
import timeit

a = open("cannyhamming.~ATH", "rb")
z = pickle.load(a)
b = open("hashhamming.~ATH", "rb")
y = pickle.load(b)

return len(z), len(y)

import numpy as np

def nxn_grid_faster(panels, full_list):

    #dictionary lookup is average O(1), list lookup is O(n)
    #ergo, store indices of panels in a dictionary to look up faster
    pandict = {} 

    for i, panel in enumerate(panels):
	pandict[panel] = i

    #initialize grid size nxn
    grid = []
    for x in range(len(panels)):
	grid.append(list(np.zeros((len(panels),), dtype=int)))

    # iterate through comparisons one time
    for compare in full_list:
	firstimg = compare[0]
	secondimg = compare[1]
	value = compare[2]

	grid[pandict[firstimg]][pandict[secondimg]] = value
	grid[pandict[secondimg]][pandict[firstimg]] = value

    return grid

import numpy as np

def nxn_grid_faster(panels, full_list):

    #dictionary lookup is average O(1), list lookup is O(n)
    #ergo, store indices of panels in a dictionary to look up faster
    pandict = {} 

    for i, panel in enumerate(panels):
	pandict[panel] = i

    #initialize grid size nxn
    grid = []
    for x in range(len(panels)):
	grid.append(list(np.zeros((len(panels),), dtype=int)))

    # iterate through comparisons one time
    for compare in full_list:
	firstimg = compare[0]
	secondimg = compare[1]
	value = compare[2]

	grid[pandict[firstimg]][pandict[secondimg]] = value
	grid[pandict[secondimg]][pandict[firstimg]] = value

    return grid
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized

import os

panels = []
for image in os.listdir('./screens/cannyfilter/'): panels.append(image)

ham = hamming_a_directory('./screens/cannyfilter/', False)

return nxn_grid_faster(panels, ham)

import cPickle as pickle
import os

import numpy as np

def nxn_grid_faster(panels, full_list):

    #dictionary lookup is average O(1), list lookup is O(n)
    #ergo, store indices of panels in a dictionary to look up faster
    pandict = {} 

    for i, panel in enumerate(panels):
	pandict[panel] = i

    #initialize grid size nxn
    grid = []
    for x in range(len(panels)):
	grid.append(list(np.zeros((len(panels),), dtype=int)))

    # iterate through comparisons one time
    for compare in full_list:
	firstimg = compare[0]
	secondimg = compare[1]
	value = compare[2]

	grid[pandict[firstimg]][pandict[secondimg]] = value
	grid[pandict[secondimg]][pandict[firstimg]] = value

    return grid

panels = os.listdir('./screens/phash/')

z = open("cannyhamming.~ATH", "rb")
#z = open("hashhamming.~ATH", "rb")
biglist = pickle.load(z)

biggrid = nxn_grid_faster(panels, biglist)

pickle_biglist = open("cannygrid.~ATH", 'wb')
#pickle_biglist = open("hashgrid.~ATH", 'wb')
pickle.dump(biggrid, pickle_biglist)
pickle_biglist.close()

import numpy as np
import os
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

def mixed_metric(edgedirectory, hashdirectory, edgeresize=True, hashresize=False):
    # find distances for canny images
    panels = os.listdir(edgedirectory)
    ham = hamming_a_directory(edgedirectory, edgeresize)

    ham_side = zip(*ham)[2]
    hmean = np.mean(ham_side)

    grid_canny = nxn_grid_from_itertools_combo(panels, map(lambda x: [x[0], x[1], x[2]/hmean], ham))

    # find distances for phash images
    ham = hamming_a_directory(hashdirectory, hashresize)

    ham_side = zip(*ham)[2]
    hmean = np.mean(ham_side)

    grid_hash = nxn_grid_from_itertools_combo(panels, ham)

    # score
    return mixed_grid(grid_canny, grid_hash, hmean)

def mixed_grid(grid_canny, grid_hash, hmean):
    mutual_grid = []
    for x, y in zip(grid_canny, grid_hash):
	z = []
	for point_a, point_b in zip(x, y):
	    z.append(int(math.floor((point_b + (point_a * hmean))/2)))
	mutual_grid.append(z)

    return mutual_grid

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import cPickle as pickle
import numpy as np

#get panel names
panels = os.listdir('./screens/cannyfilter/')

#load grids
z = open("cannygrid.~ATH", "rb")
cannygrid = pickle.load(z)

cannymean = np.mean(cannygrid)

for m, x in enumerate(cannygrid):
    for i, pt in enumerate(x):
	cannygrid[m][i] = pt/cannymean

y = open("hashgrid.~ATH", "rb")
hashgrid = pickle.load(y)

#find mixed grid
grid = mixed_grid(cannygrid, hashgrid, np.mean(hashgrid))

#find pca
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

#cluster
lowclust = 4
highclust = 300
clust = cluster(principalComponents, lowclust, highclust, 100, 1)

#attach names
names = give_names(clust[2], panels, principalComponents)

#visualize
plt.plot(range(lowclust, len(clust[0])+lowclust), clust[0])
plt.title('SSE vs K - Image Hashes using PCA')
plt.xlabel('Value of K')
plt.ylabel('SSE')
plt.show()

#move to folders
for i, x in enumerate(names):
    path = "./screens/img/clustered/" + str(i) + "/"
    for y in x: move_directory("./screens/img/", path, y)

import numpy as np
import os
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

def mixed_metric(edgedirectory, hashdirectory, edgeresize=True, hashresize=False):
    # find distances for canny images
    panels = os.listdir(edgedirectory)
    ham = hamming_a_directory(edgedirectory, edgeresize)

    ham_side = zip(*ham)[2]
    hmean = np.mean(ham_side)

    grid_canny = nxn_grid_from_itertools_combo(panels, map(lambda x: [x[0], x[1], x[2]/hmean], ham))

    # find distances for phash images
    ham = hamming_a_directory(hashdirectory, hashresize)

    ham_side = zip(*ham)[2]
    hmean = np.mean(ham_side)

    grid_hash = nxn_grid_from_itertools_combo(panels, ham)

    # score
    return mixed_grid(grid_canny, grid_hash, hmean)

def mixed_grid(grid_canny, grid_hash, hmean):
    mutual_grid = []
    for x, y in zip(grid_canny, grid_hash):
	z = []
	for point_a, point_b in zip(x, y):
	    z.append(int(math.floor((point_b + (point_a * hmean))/2)))
	mutual_grid.append(z)

    return mutual_grid

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import cPickle as pickle
import numpy as np

#get panel names
panels = os.listdir('./screens/cannyfilter/')

#load grids
z = open("cannygrid.~ATH", "rb")
cannygrid = pickle.load(z)
grid = filter_grid(cannygrid)

#find pca
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(grid).tolist()

#cluster
lowclust = 20
highclust = 200
clust = cluster(principalComponents, lowclust, highclust, 100, 1)

#attach names
names = give_names(clust[2], panels, principalComponents)

#move to folders
for i, x in enumerate(names):
    path = "./screens/img/clustered/" + str(i) + "/"
    for y in x: move_directory("./screens/img/", path, y)

import networkx as nx
from networkx.algorithms import community
import os
import numpy as np
import cPickle as pickle
import itertools
import numpy as np

def nxn_grid_faster(panels, full_list):

    #dictionary lookup is average O(1), list lookup is O(n)
    #ergo, store indices of panels in a dictionary to look up faster
    pandict = {} 

    for i, panel in enumerate(panels):
	pandict[panel] = i

    #initialize grid size nxn
    grid = []
    for x in range(len(panels)):
	grid.append(list(np.zeros((len(panels),), dtype=int)))

    # iterate through comparisons one time
    for compare in full_list:
	firstimg = compare[0]
	secondimg = compare[1]
	value = compare[2]

	grid[pandict[firstimg]][pandict[secondimg]] = value
	grid[pandict[secondimg]][pandict[firstimg]] = value

    return grid
def shorthand_grid(grid, threshold):

    newgrid = []
    for img in grid:
	shorthand = []
	for i, point in enumerate(img):
	    if point < threshold:
		shorthand.append(i)
	newgrid.append(shorthand)

    return newgrid
def difference_lists(list1, list2):
    return len(list(set(list1).symmetric_difference(set(list2))))

def most_similar_lists(listoflists):
    minsize = 9999
    choice1 = []
    choice2 = []

    for i, x in listoflists:
	for j, y in listoflists:
	    if i < j:
		thissize = difference_lists(x,y)
		if thissize < minsize:
		    minsize = thissize
		    choice1 = x
		    choice2 = y

    return [choice1, choice2]
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/phash/'

panels = os.listdir(directory)

a = open("hashgrid.~ATH", 'rb')
grid = pickle.load(a)

threshold = 16

short = shorthand_grid(grid, threshold)

G = nx.Graph()

#add nodes
G.add_nodes_from(panels)

#add edges
for i, eg in enumerate(short):
    for val in eg:
	#ignore self edges since they're not really being used
	if i != val:
	    weight = threshold-grid[i][val]+1
	    G.add_edge(panels[i], panels[val], weight=weight)

#delete vertices with no edges
for x in list(G.nodes):
    if len(G.adj[x]) == 0:
	G.remove_node(x)

clust = community.greedy_modularity_communities(G)

folname = 0
for frzset in clust:
    val = list(frzset)
    if len(val) > 1:
	for x in val:
	    move_directory('./screens/img/', './screens/img/clustered/' + str(folname) + '/', panels[x])
    folname += 1

import numpy as np
import os
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

def mixed_metric(edgedirectory, hashdirectory, edgeresize=True, hashresize=False):
    # find distances for canny images
    panels = os.listdir(edgedirectory)
    ham = hamming_a_directory(edgedirectory, edgeresize)

    ham_side = zip(*ham)[2]
    hmean = np.mean(ham_side)

    grid_canny = nxn_grid_from_itertools_combo(panels, map(lambda x: [x[0], x[1], x[2]/hmean], ham))

    # find distances for phash images
    ham = hamming_a_directory(hashdirectory, hashresize)

    ham_side = zip(*ham)[2]
    hmean = np.mean(ham_side)

    grid_hash = nxn_grid_from_itertools_combo(panels, ham)

    # score
    return mixed_grid(grid_canny, grid_hash, hmean)

def mixed_grid(grid_canny, grid_hash, hmean):
    mutual_grid = []
    for x, y in zip(grid_canny, grid_hash):
	z = []
	for point_a, point_b in zip(x, y):
	    z.append(int(math.floor((point_b + (point_a * hmean))/2)))
	mutual_grid.append(z)

    return mutual_grid

import random, math
import numpy as np
from joblib import Parallel, delayed
import multiprocessing
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


# returns an integer representing distance between two points
def find_distance(x,y):
    #use hamming distance if greater than R^10, otherwise use Euclidian Distance
    if len(x) > 10:
	return hamming(quantize(x), quantize(y))
    else:
	#sqrt ( (a-x)^2 + (b-y)^2 + (c-z)^2 ... )
	distance = 0
	for origin, destination in zip(x,y):
	    distance += (origin-destination)**2

	return math.sqrt(distance)

# A single attempt at clustering data into K points
def kmeans_clustering(matrix, k):
    #init k random points
    points = random.sample(matrix, k)

    #placeholders
    oldclusters = [-1]
    clusters = []
    for i in range(k):
	clusters.append([])
    emptyclusters = clusters

    #loop until points don't change
    while(oldclusters != clusters):
	oldclusters = clusters
	clusters = emptyclusters #use space instead of time to avoid iterating to zero out every loop

	#group all data points to nearest point
	for x in matrix:
	    distances = []

	    for y in points:
		distances.append(find_distance(x,y))

	    clusters[distances.index(min(distances))].append(x)

	#when all points are grouped, calculate new mean for each point
	for i, cluster in enumerate(clusters):
	    if cluster:
		points[i] = map(np.mean, zip(*cluster))

    return clusters

# run K-means a few times, return clustering with minimum intracluster variance
def cluster_given_K(matrix, k, n=25):
    clusterings = []

    # run k-means a few times
    for x in range(n):
	clusterings.append(kmeans_clustering(matrix, k))

    # calculate intracluster variance for each clustering
    ##  this is just the sum of all distances from every point to it's cluster's center
    distances = []
    for clustering in clusterings:
	variance = 0
	for cluster in clustering:
	    center = map(np.mean, zip(*cluster))
	    for point in cluster:
		variance += find_distance(point,center)

	distances.append(variance)

    return [clusterings[distances.index(min(distances))], min(distances)]

# Loosely look for the elbow in the graph
def elbowmethod(candidates, debug_flag=0):
    varscores = zip(*candidates)[1]

    #just for debug purposes
    if debug_flag == 1
	return varscores

    percentages = map(lambda x: 1-(x/varscores[0]), varscores)

    elbowseek = []

    for point in range(0,len(percentages)-1):
	if point is 0:
	    elbowseek.append(0)
	elif point is len(percentages)-1:
	    elbowseek.append(percentages[point]-percentages[point-1])
	else:
	    elbowseek.append((percentages[point]-percentages[point-1]) - \
			       (percentages[point+1]-percentages[point]))

    return elbowseek

# Runs cluster_given_K multiple times, for each value of K
def cluster(matrix, minK=1, maxK=-1, runs=50, debug_flag=0):
    if not matrix:
	return []
    if maxK is -1:
	maxK = len(matrix)

    num_cores = multiprocessing.cpu_count()
    candidates = Parallel(n_jobs=num_cores)(delayed(cluster_given_K)(matrix, x, runs) for x in range(minK, maxK))

    elbowseek = elbowmethod(candidates, debug_flag)

    if debug_flag == 1:
	return elbowseek, candidates, candidates[elbowseek.index(max(elbowseek))][0]

    return candidates[elbowseek.index(max(elbowseek))][0]

def give_names(clustering, names, grid):
    ret = []

    for x in clustering:
	ret_a = []
	for y in x:
	    ret_a.append(names[grid.index(y)])
	ret.append(ret_a)

    return ret
import numpy as np

def nxn_grid_faster(panels, full_list):

    #dictionary lookup is average O(1), list lookup is O(n)
    #ergo, store indices of panels in a dictionary to look up faster
    pandict = {} 

    for i, panel in enumerate(panels):
	pandict[panel] = i

    #initialize grid size nxn
    grid = []
    for x in range(len(panels)):
	grid.append(list(np.zeros((len(panels),), dtype=int)))

    # iterate through comparisons one time
    for compare in full_list:
	firstimg = compare[0]
	secondimg = compare[1]
	value = compare[2]

	grid[pandict[firstimg]][pandict[secondimg]] = value
	grid[pandict[secondimg]][pandict[firstimg]] = value

    return grid
def shorthand_grid(grid, threshold):

    newgrid = []
    for img in grid:
	shorthand = []
	for i, point in enumerate(img):
	    if point < threshold:
		shorthand.append(i)
	newgrid.append(shorthand)

    return newgrid
def difference_lists(list1, list2):
    return len(list(set(list1).symmetric_difference(set(list2))))

def most_similar_lists(listoflists):
    minsize = 9999
    choice1 = []
    choice2 = []

    for i, x in listoflists:
	for j, y in listoflists:
	    if i < j:
		thissize = difference_lists(x,y)
		if thissize < minsize:
		    minsize = thissize
		    choice1 = x
		    choice2 = y

    return [choice1, choice2]
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import cPickle as pickle
import numpy as np
import networkx as nx
from networkx.algorithms import community
import os

#get panel names
directory = './screens/img/'
panels = os.listdir(directory)

#load grids
z = open("cannygrid.~ATH", "rb")
cannygrid = pickle.load(z)

cannymean = np.mean(cannygrid)

for m, x in enumerate(cannygrid):
    for i, pt in enumerate(x):
	cannygrid[m][i] = pt/cannymean

y = open("hashgrid.~ATH", "rb")
hashgrid = pickle.load(y)

#find mixed grid
grid = mixed_grid(cannygrid, hashgrid, np.mean(hashgrid))    

threshold = 16

short = shorthand_grid(grid, threshold)

#represent as graph
G = nx.Graph()

#add nodes
G.add_nodes_from(panels)

#add edges
for i, eg in enumerate(short):
    for val in eg:
	#ignore self edges since they're not really being used
	if i != val:
	    weight = threshold-grid[i][val]+1
	    G.add_edge(panels[i], panels[val], weight=weight)

#delete vertices with no edges
for x in list(G.nodes):
    if len(G.adj[x]) == 0:
	G.remove_node(x)

Clust = community.greedy_modularity_communities(G)

folname = 0
for frzset in clust:
    val = list(frzset)
    if len(val) > 1:
	for x in val:
	    move_directory('./screens/img/', './screens/img/clustered/' + str(folname) + '/', x)
    folname += 1

import os
from PIL import Image

def weirdpixels(directory, panels):

    small_score = []
    for panel in panels:
	xscore = []
	xval = []
	img = Image.open(directory+panel)
	imgdata = list(img.getdata())

	for pixel in imgdata:
	    if pixel in xval:
		xscore[xval.index(pixel)] += 1
	    else:
		xval.append(pixel)
		xscore.append(1)

	if len(xscore) == 1:
	    small_score.append(0) #completely black image should have threshold of 0
	else:
	    small_score.append(xscore[xscore.index(min(xscore))])

    return small_score

import os
from PIL import Image

def weirdpixels(directory, panels):

    small_score = []
    for panel in panels:
	xscore = []
	xval = []
	img = Image.open(directory+panel)
	imgdata = list(img.getdata())

	for pixel in imgdata:
	    if pixel in xval:
		xscore[xval.index(pixel)] += 1
	    else:
		xval.append(pixel)
		xscore.append(1)

	if len(xscore) == 1:
	    small_score.append(0) #completely black image should have threshold of 0
	else:
	    small_score.append(xscore[xscore.index(min(xscore))])

    return small_score
directory = './screens/cannyfilter/'
panels = os.listdir(directory)
return weirdpixels(directory, panels)

def shorthand_grid_scaled(grid, thresholds):

    newgrid = []
    for img, threshold in zip(grid, thresholds):
	shorthand = []
	for i, point in enumerate(img):
	    if point < threshold:
		shorthand.append(i)
	newgrid.append(shorthand)

    return newgrid

def shorthand_grid_scaled(grid, thresholds):

    newgrid = []
    for img, threshold in zip(grid, thresholds):
	shorthand = []
	for i, point in enumerate(img):
	    if point < threshold:
		shorthand.append(i)
	newgrid.append(shorthand)

    return newgrid

grid = [[1,2,3],[1,1,3],[35,11,22]]
thresholds = [2,2,20]

return shorthand_grid_scaled(grid,thresholds)

import networkx as nx
from networkx.algorithms import community
import os
import numpy as np
import cPickle as pickle
import itertools
import numpy as np

def nxn_grid_faster(panels, full_list):

    #dictionary lookup is average O(1), list lookup is O(n)
    #ergo, store indices of panels in a dictionary to look up faster
    pandict = {} 

    for i, panel in enumerate(panels):
	pandict[panel] = i

    #initialize grid size nxn
    grid = []
    for x in range(len(panels)):
	grid.append(list(np.zeros((len(panels),), dtype=int)))

    # iterate through comparisons one time
    for compare in full_list:
	firstimg = compare[0]
	secondimg = compare[1]
	value = compare[2]

	grid[pandict[firstimg]][pandict[secondimg]] = value
	grid[pandict[secondimg]][pandict[firstimg]] = value

    return grid
def shorthand_grid_scaled(grid, thresholds):

    newgrid = []
    for img, threshold in zip(grid, thresholds):
	shorthand = []
	for i, point in enumerate(img):
	    if point < threshold:
		shorthand.append(i)
	newgrid.append(shorthand)

    return newgrid
import os
from PIL import Image

def weirdpixels(directory, panels):

    small_score = []
    for panel in panels:
	xscore = []
	xval = []
	img = Image.open(directory+panel)
	imgdata = list(img.getdata())

	for pixel in imgdata:
	    if pixel in xval:
		xscore[xval.index(pixel)] += 1
	    else:
		xval.append(pixel)
		xscore.append(1)

	if len(xscore) == 1:
	    small_score.append(0) #completely black image should have threshold of 0
	else:
	    small_score.append(xscore[xscore.index(min(xscore))])

    return small_score
def difference_lists(list1, list2):
    return len(list(set(list1).symmetric_difference(set(list2))))

def most_similar_lists(listoflists):
    minsize = 9999
    choice1 = []
    choice2 = []

    for i, x in listoflists:
	for j, y in listoflists:
	    if i < j:
		thissize = difference_lists(x,y)
		if thissize < minsize:
		    minsize = thissize
		    choice1 = x
		    choice2 = y

    return [choice1, choice2]
import numpy as np
import os
from PIL import Image
import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized


def nxn_grid_from_itertools_combo(panels, full_list):
    # Create nxn grid such that x,y is a comparison between panel x and panel y
    #   this is the format that you'd get if you did every comparison but we used itertools
    #   be more efficient. Now that we need these comparisons in a matrix we need to convert it.
    grid = []

    for image1 in panels:
	compare_img1 = []
	for image2 in panels:
	    if image1 == image2:
		compare_img1.append(0)
	    else:
		val = [x[2] for x in full_list if ((x[0] == image1 and x[1] == image2) or \
					    (x[0] == image2 and x[1] == image2))]
		if val:
		    compare_img1.append(val[0])
		else:
		    compare_img1.append(grid[panels.index(image2)][panels.index(image1)])

	grid.append(compare_img1)

    return grid

def filter_out_nonduplicates(directory, resize=True):
    ## Perform comparisons without duplicates

    full_list = hamming_a_directory(directory, resize)

    ## convert comparisons to an nxn grid, as if we had duplicates

    # Create list of panels
    panels = os.listdir(directory)

    # Create nxn grid such that x,y is a comparison between panel x and panel y
    nxn_grid = nxn_grid_from_itertools_combo(panels, full_list)

    # find mu and sigma of each panel compared to each other panel, filter out probable matches
    return filter_grid(nxn_grid)

def filter_grid(grid):
    gmean = np.mean(grid)
    gstd = np.std(grid)

    filtered_grid = []
    for i, panel in enumerate(grid):
	for x, comparison in enumerate(panel):
	    flag = False
	    if i != x:
		if comparison < (gmean-gstd):
		    flag = True
		    break
	if flag == True:
	    filtered_grid.append(panel)
    return filtered_grid

def move_directory(imgsrc, directory, filename):
    if not os.path.exists(directory):
	os.makedirs(directory)
    try:
	newfile = Image.open(imgsrc+filename)
	newfile.save(directory+filename)
    except:        
	newfile = Image.open(imgsrc+filename[:-3]+"gif")
	newfile.save(directory+filename[:-3]+"gif")

directory = './screens/cannyfilter/'

panels = os.listdir(directory)

a = open("cannyhamming.~ATH", 'rb')
grid = nxn_grid_faster(panels, pickle.load(a))

edgepixel_vals = weirdpixels(directory, panels)

thresholds = map(lambda x: min(x*0.95, 350), edgepixel_vals)

short = shorthand_grid_scaled(grid, thresholds)

G = nx.Graph()

#add nodes
G.add_nodes_from(panels)

#add edges
for i, eg in enumerate(short):
    for val in eg:
	#ignore self edges since they're not really being used
	if i != val:
	    weight = thresholds[i]-grid[i][val]+1
	    G.add_edge(panels[i], panels[val], weight=weight)

#delete vertices with no edges
for x in list(G.nodes):
    if len(G.adj[x]) == 0:
	G.remove_node(x)

clust = community.greedy_modularity_communities(G)

folname = 0
for frzset in clust:
    val = list(frzset)
    if len(val) > 1:
	for x in val:
	    move_directory('./screens/img/', './screens/img/clustered/' + str(folname) + '/', x)
    folname += 1

import PIL
from PIL import Image
import io, itertools, os
from joblib import Parallel, delayed
import multiprocessing
import numpy as np

def hamming(x, y):
    if len(x) == len(y):
	#Choosing the distance between the image or the image's inverse, whichever is closer
	return min(sum(c1 != c2 for c1, c2 in zip(x, y)), sum(c1 == c2 for c1, c2 in zip(x, y)))
    else:
	return -1

def compare_img(image1, image2, dire, resize):
    i1 = Image.open(dire + image1)
    if resize:
	i1 = i1.resize((100,100))

    i1_b = i1.tobytes()

    i2 = Image.open(dire + image2)
    if resize:
	i2 = i2.resize((100,100))

    i2_b = i2.tobytes()

    dist = hamming(i1_b, i2_b)
    return dist

#including here a helper function so I can call a function in parallel
def output_format(image1, image2, dire, resize):
    return [image1, image2, compare_img(image1, image2, dire, resize)]

def hamming_a_directory(dire, resize=True):
    num_cores = multiprocessing.cpu_count()    
    return Parallel(n_jobs=num_cores)(delayed(output_format)(image1, image2, dire, resize)\
			       for image1, image2 in itertools.combinations(os.listdir(dire), 2))

def quantize(img_arr, dimx=8, dimy=8):
    quantized = []
    for x in img_arr:
	if x >= np.mean(img_arr): quantized.append(255)
	else: quantized.append(0)
    return quantized

import cv2 as cv

fromdir = './screens/img/'
todir = './screens/cannyfilter/'

for image in os.listdir(fromdir):

    if not os.path.isfile(fromdir+image):
	continue

    #gif -> png for opencv compatability
    im_gif = Image.open(fromdir+image)
    im_gif = im_gif.resize((100,100))
    saveto = todir + image.split(".")[0] + ".png"
    im_gif.save(saveto)

    #Canny Edge Detection, overwrite png
    img_orig = cv.imread(saveto, 1)
    edges = cv.Canny(img_orig,100,200)
    img_new = Image.fromarray(edges)
    img_new.save(saveto)