phyks
/
bloomysearch
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Dropping jQuery !

This commit is contained in:
Phyks 2014-01-11 00:19:22 +01:00
parent d759e7c8ab
commit 0c33477699
5 changed files with 49 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*/__pycache__
*/search_index

7
index.html
View File

@ -7,14 +7,13 @@
<body> <body>
<h1>Bloom.JS demo</h1> <h1>Bloom.JS demo</h1>
<p id="loading"></p> <p id="loading"></p>
<form> <form id="search_form">
<p> <p>
<input type="text" id="search" name="search" value="Search for articles..."/> <input type="text" id="search" name="search" value="Search for articles..."/>
</p> </p>
</form> </form>
<div id="results"></div> <div id="results"></div>
<script type="text/javascript" src="jquery-2.0.3.min.js"></script> <script type="text/javascript" src="js/bloomfilter.js"></script>
<script type="text/javascript" src="bloomfilter.js"></script> <script type="text/javascript" src="js/bloom.js"></script>
<script type="text/javascript" src="bloom.js"></script>
</body> </body>
</html> </html>

16
index_generation/generate_index.py
View File

@ -4,6 +4,7 @@ import os
from lxml import html from lxml import html
import re import re
import stemmer import stemmer
from bitarray import bitarray
from pybloom import BloomFilter from pybloom import BloomFilter
@ -20,10 +21,15 @@ def remove_common_words(words):
returned = [word for word in words if len(word) > 3] returned = [word for word in words if len(word) > 3]
return returned return returned
def bitfield(n):
return [1 if digit=='1' else 0 for digit in bin(n)[2:]]
# ============================================================================= # =============================================================================
samples = list_directory("samples/") samples = list_directory("../samples/")
filters = {} filters = {}
p = stemmer.PorterStemmer() p = stemmer.PorterStemmer()
write = bitarray()
for sample in samples: for sample in samples:
with open(sample, 'r') as sample_fh: with open(sample, 'r') as sample_fh:
@ -44,5 +50,9 @@ for sample in samples:
for word in words: for word in words:
filters[sample].add(word) filters[sample].add(word)
print(sum(len(filter.bitarray.tobytes()) for filter in filters.values()) / with open('search_index', 'wb') as index_fh:
len(filters)) index_fh.write(filters[samples[0]].bitarray.tobytes()) # TODO
write.extend(bitfield(len(filters[samples[0]].bitarray)))
write.extend(filters[samples[0]].bitarray)
print(write)

56
js/bloom.js
View File

@ -2,34 +2,40 @@ var loading = false;
var usable = false; var usable = false;
var index = false; var index = false;
$("form").submit(function(e) { window.onload = function() {
};
document.getElementById('search_form').addEventListener('submit', function(e) {
e.preventDefault(); e.preventDefault();
}); });
$("#search").click(function() { document.getElementById('search').addEventListener('click', function() {
if($(this).val() == "Search for articles...") { if(this.value == "Search for articles...") {
$(this).val(""); this.value = "";
} }
if(index === false) { if(index === false) {
loading = true; loading = true;
$("#loading").text("Loading index file..."); document.getElementById("loading").innerHTML = "Loading index file...";
$.getJSON("index.json", function(data) {
loading = false;
usable = true;
$("#loading").text("");
index = new Array();
for(var key in data) { var oReq = new XMLHttpRequest();
index[key] = new BloomFilter(32*256, 16); oReq.open("GET", "/index_generation/search_index", true);
oReq.responseType = "arraybuffer";
for(var word_index in data[key]) { oReq.onload = function (oEvent) {
index[key].add(data[key][word_index]); var arrayBuffer = oReq.response; // Note: not oReq.responseText
if (arrayBuffer) {
loading = false;
usable = true;
document.getElementById("loading").innerHTML = "";
var tmp = new Uint8Array(arrayBuffer);
for (var i = 0; i < tmp.byteLength; i++) {
// TODO
} }
} }
};
callback_change(); oReq.send(null);
});
} }
}); });
@ -37,16 +43,16 @@ function callback_change() {
if(!usable) { if(!usable) {
return; return;
} }
var search = $("#search").val(); var search = document.getElementById("search").value;
$("#results").html("<h2>Results :</h2>"); document.getElementById("results").innerHTML = "<h2>Results :</h2>";
for(var key in index) { for(var key in index) {
if(index[key].test(search)) { //if(index[key].test(search)) { TODO
$("#results").append("<p>"+key+"</p>"); document.getElementById("results").innerHTML += "<p>"+key+"</p>";
} //}
} }
if(!$("#results p").length) { if(!document.querySelectorAll("#results p").length) {
$("#results").append("<p>No results...</p>"); document.getElementById("results").innerHTML += "<p>No results...</p>";
} }
} }
$("#search").on('input', callback_change); document.getElementById("search").addEventListener('input', callback_change);

6
js/jquery-2.0.3.min.js vendored
View File

File diff suppressed because one or more lines are too long