Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

radisk file hashing #1005

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

radisk file hashing #1005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
24b6733
add: radisk file hashing
sirpy Aug 31, 2020
82d83c1
add: upgrading old format
sirpy Aug 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 58 additions & 10 deletions lib/rfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,51 @@ function Store(opt){
opt.log = opt.log || console.log;
opt.file = String(opt.file || 'radata');
var fs = require('fs'), u;

var sha256 = require('./sha256');
var store = function Store(){};
if(Store[opt.file]){
console.log("Warning: reusing same fs store and options as 1st.");
return Store[opt.file];
}

if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) }

const indexFile = opt.file+'/'+opt.file+'.idx'
var logger = fs.createWriteStream(indexFile, {
flags: 'a' // 'a' means appending (old data will be preserved)
})
let cleanupCalled = false
const cleanupIndex = (err) => {
if(cleanupCalled) return;
cleanupCalled = true;
logger.end(() => {
process.exit()
})
}
[`exit`, `SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => {
process.on(eventType, cleanupIndex);
})
var index = {}

Store[opt.file] = store;
var puts = {};

// TODO!!! ADD ZLIB INFLATE / DEFLATE COMPRESSION!
store.put = function(file, data, cb){
puts[file] = data;
var random = Math.random().toString(36).slice(-3);
var tmp = opt.file+'-'+file+'-'+random+'.tmp';
const hash = sha256.hash(file);
var tmp = opt.file+'-'+hash+'-'+random+'.tmp';
fs.writeFile(tmp, data, function(err, ok){
delete puts[file];
if(err){ return cb(err) }
move(tmp, opt.file+'/'+file, cb);
move(tmp, file,hash, cb);
});
};
store.get = function(file, cb){ var tmp; // this took 3s+?
if(tmp = puts[file]){ cb(u, tmp); return }
fs.readFile(opt.file+'/'+file, function(err, data){
const hash = sha256.hash(file);
fs.readFile(opt.file+'/'+hash, function(err, data){
if(err){
if('ENOENT' === (err.code||'').toUpperCase()){
return cb();
Expand All @@ -36,9 +58,18 @@ function Store(opt){
});
};

if(!fs.existsSync(opt.file)){ fs.mkdirSync(opt.file) }

function move(oldPath, newPath, cb) {
function move(oldPath, file, hash, cb) {
const newPath = opt.file+'/'+hash;
if(!index[hash])
{
index[hash] = true;

logger.write(file+'\n',function(err){
if(err)
cb(err)
})
}
fs.rename(oldPath, newPath, function (err) {
if (err) {
if (err.code === 'EXDEV') {
Expand All @@ -63,10 +94,27 @@ function Store(opt){
};

store.list = function(cb, match, params, cbs){
var dir = fs.readdirSync(opt.file);
dir.forEach(function(file){
cb(file);
})
if(fs.existsSync(indexFile))
{
fs.readFileSync(indexFile,'utf8').split(/\r?\n/).forEach(fn => {
//skip the empty line
if(!fn) return;
index[sha256.hash(fn)] = true
cb(fn)
});
}
//try to perform upgrade if index is empty
if(Object.keys(index).length === 0) {
const re = /^[0-9A-Fa-f]+$/g;
const dir = fs.readdirSync(opt.file);
dir.forEach(function(fn){
//convert only non 32byte hex filenames (ie not in new format)
if(!fn || fn === opt.file+'.idx' || (fn.length===64 && re.test(fn)) ) return;
const hash = sha256.hash(fn);
move(opt.file+"/"+fn,fn,hash,() => {})
cb(fn)
})
}
cb();
};

Expand Down
180 changes: 180 additions & 0 deletions lib/sha256.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* SHA-256 (FIPS 180-4) implementation in JavaScript (c) Chris Veness 2002-2019 */
/* MIT Licence */
/* www.movable-type.co.uk/scripts/sha256.html */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */


/**
* SHA-256 hash function reference implementation.
*
* This is an annotated direct implementation of FIPS 180-4, without any optimisations. It is
* intended to aid understanding of the algorithm rather than for production use.
*
* While it could be used where performance is not critical, I would recommend using the ‘Web
* Cryptography API’ (developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) for the browser,
* or the ‘crypto’ library (nodejs.org/api/crypto.html#crypto_class_hash) in Node.js.
*
* See csrc.nist.gov/groups/ST/toolkit/secure_hashing.html
* csrc.nist.gov/groups/ST/toolkit/examples.html
*/
class Sha256 {

/**
* Generates SHA-256 hash of string.
*
* @param {string} msg - (Unicode) string to be hashed.
* @param {Object} [options]
* @param {string} [options.msgFormat=string] - Message format: 'string' for JavaScript string
* (gets converted to UTF-8 for hashing); 'hex-bytes' for string of hex bytes ('616263' ≡ 'abc') .
* @param {string} [options.outFormat=hex] - Output format: 'hex' for string of contiguous
* hex bytes; 'hex-w' for grouping hex bytes into groups of (4 byte / 8 character) words.
* @returns {string} Hash of msg as hex character string.
*
* @example
* import Sha256 from './sha256.js';
* const hash = Sha256.hash('abc'); // 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'
*/
static hash(msg, options) {
const defaults = { msgFormat: 'string', outFormat: 'hex' };
const opt = Object.assign(defaults, options);

// note use throughout this routine of 'n >>> 0' to coerce Number 'n' to unsigned 32-bit integer

switch (opt.msgFormat) {
default: // default is to convert string to UTF-8, as SHA only deals with byte-streams
case 'string': msg = utf8Encode(msg); break;
case 'hex-bytes':msg = hexBytesToString(msg); break; // mostly for running tests
}

// constants [§4.2.2]
const K = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 ];

// initial hash value [§5.3.3]
const H = [
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 ];

// PREPROCESSING [§6.2.1]

msg += String.fromCharCode(0x80); // add trailing '1' bit (+ 0's padding) to string [§5.1.1]

// convert string msg into 512-bit blocks (array of 16 32-bit integers) [§5.2.1]
const l = msg.length/4 + 2; // length (in 32-bit integers) of msg + ‘1’ + appended length
const N = Math.ceil(l/16); // number of 16-integer (512-bit) blocks required to hold 'l' ints
const M = new Array(N); // message M is N×16 array of 32-bit integers

for (let i=0; i<N; i++) {
M[i] = new Array(16);
for (let j=0; j<16; j++) { // encode 4 chars per integer (64 per block), big-endian encoding
M[i][j] = (msg.charCodeAt(i*64+j*4+0)<<24) | (msg.charCodeAt(i*64+j*4+1)<<16)
| (msg.charCodeAt(i*64+j*4+2)<< 8) | (msg.charCodeAt(i*64+j*4+3)<< 0);
} // note running off the end of msg is ok 'cos bitwise ops on NaN return 0
}
// add length (in bits) into final pair of 32-bit integers (big-endian) [§5.1.1]
// note: most significant word would be (len-1)*8 >>> 32, but since JS converts
// bitwise-op args to 32 bits, we need to simulate this by arithmetic operators
const lenHi = ((msg.length-1)*8) / Math.pow(2, 32);
const lenLo = ((msg.length-1)*8) >>> 0;
M[N-1][14] = Math.floor(lenHi);
M[N-1][15] = lenLo;


// HASH COMPUTATION [§6.2.2]

for (let i=0; i<N; i++) {
const W = new Array(64);

// 1 - prepare message schedule 'W'
for (let t=0; t<16; t++) W[t] = M[i][t];
for (let t=16; t<64; t++) {
W[t] = (Sha256.σ1(W[t-2]) + W[t-7] + Sha256.σ0(W[t-15]) + W[t-16]) >>> 0;
}

// 2 - initialise working variables a, b, c, d, e, f, g, h with previous hash value
let a = H[0], b = H[1], c = H[2], d = H[3], e = H[4], f = H[5], g = H[6], h = H[7];

// 3 - main loop (note '>>> 0' for 'addition modulo 2^32')
for (let t=0; t<64; t++) {
const T1 = h + Sha256.Σ1(e) + Sha256.Ch(e, f, g) + K[t] + W[t];
const T2 = Sha256.Σ0(a) + Sha256.Maj(a, b, c);
h = g;
g = f;
f = e;
e = (d + T1) >>> 0;
d = c;
c = b;
b = a;
a = (T1 + T2) >>> 0;
}

// 4 - compute the new intermediate hash value (note '>>> 0' for 'addition modulo 2^32')
H[0] = (H[0]+a) >>> 0;
H[1] = (H[1]+b) >>> 0;
H[2] = (H[2]+c) >>> 0;
H[3] = (H[3]+d) >>> 0;
H[4] = (H[4]+e) >>> 0;
H[5] = (H[5]+f) >>> 0;
H[6] = (H[6]+g) >>> 0;
H[7] = (H[7]+h) >>> 0;
}

// convert H0..H7 to hex strings (with leading zeros)
for (let h=0; h<H.length; h++) H[h] = ('00000000'+H[h].toString(16)).slice(-8);

// concatenate H0..H7, with separator if required
const separator = opt.outFormat=='hex-w' ? ' ' : '';

return H.join(separator);

/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

function utf8Encode(str) {
try {
return new TextEncoder().encode(str, 'utf-8').reduce((prev, curr) => prev + String.fromCharCode(curr), '');
} catch (e) { // no TextEncoder available?
return unescape(encodeURIComponent(str)); // monsur.hossa.in/2012/07/20/utf-8-in-javascript.html
}
}

function hexBytesToString(hexStr) { // convert string of hex numbers to a string of chars (eg '616263' -> 'abc').
const str = hexStr.replace(' ', ''); // allow space-separated groups
return str=='' ? '' : str.match(/.{2}/g).map(byte => String.fromCharCode(parseInt(byte, 16))).join('');
}
}



/**
* Rotates right (circular right shift) value x by n positions [§3.2.4].
* @private
*/
static ROTR(n, x) {
return (x >>> n) | (x << (32-n));
}


/**
* Logical functions [§4.1.2].
* @private
*/
static Σ0(x) { return Sha256.ROTR(2, x) ^ Sha256.ROTR(13, x) ^ Sha256.ROTR(22, x); }
static Σ1(x) { return Sha256.ROTR(6, x) ^ Sha256.ROTR(11, x) ^ Sha256.ROTR(25, x); }
static σ0(x) { return Sha256.ROTR(7, x) ^ Sha256.ROTR(18, x) ^ (x>>>3); }
static σ1(x) { return Sha256.ROTR(17, x) ^ Sha256.ROTR(19, x) ^ (x>>>10); }
static Ch(x, y, z) { return (x & y) ^ (~x & z); } // 'choice'
static Maj(x, y, z) { return (x & y) ^ (x & z) ^ (y & z); } // 'majority'

}


/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

module.exports = Sha256;
12 changes: 12 additions & 0 deletions test/rad/rad.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,18 @@ var names = ["Adalard","Adora","Aia","Albertina","Alfie","Allyn","Amabil","Ammam

});

describe("RAD file hash",function(){
it("should have index file",function(done){
let found = false
opt.store.list((fn) => {
if(fn !== undefined) { found = true; return}
else {
expect(found).to.be.ok()
done()
}
})
})
})
var ntmp = names;
describe('RAD + GUN', function(){
var ochunk = 1000;
Expand Down