From e6a1c6b5bbe54a7c4c663e0c950febadd1ab184c Mon Sep 17 00:00:00 2001 From: pasta Date: Tue, 7 Jul 2020 16:03:07 -0500 Subject: [PATCH] scripted-diff: change all tabs to spaces in linearize-hashes.py and linearize-data.py It appears bitcoin has already done this previously(seemingly forever), however, we for some reason have tabs. This was causing the linter to fail because a backport touched lines which resulted in linter yelling at me for adding lines beginning with tabs. To fix this, replacing all tabs with spaces in these two files. -BEGIN VERIFY SCRIPT- sed -i 's/\t/ /g' contrib/linearize/linearize-*.py -END VERIFY SCRIPT- Signed-off-by: pasta --- contrib/linearize/linearize-data.py | 474 +++++++++++++------------- contrib/linearize/linearize-hashes.py | 214 ++++++------ 2 files changed, 344 insertions(+), 344 deletions(-) diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index f1fd8f92ce..af0c8daa68 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -23,302 +23,302 @@ settings = {} ##### Switch endian-ness ##### def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() + """ Switches the endianness of a hex string (in pairs of hex chars) """ + pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] + return b''.join(pairList[::-1]).decode() def uint32(x): - return x & 0xffffffff + return x & 0xffffffff def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) + return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | + (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) + out_words = [] + for i in range(0, len(in_buf), 4): + word = struct.unpack('@I', in_buf[i:i+4])[0] + out_words.append(struct.pack('@I', bytereverse(word))) + return b''.join(out_words) def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) + out_words = [] + for i in range(0, len(in_buf), 4): + out_words.append(in_buf[i:i+4]) + out_words.reverse() + return b''.join(out_words) def calc_hdr_hash(blk_hdr): - #hash1 = hashlib.sha256() - #hash1.update(blk_hdr) - #hash1_o = hash1.digest() + #hash1 = hashlib.sha256() + #hash1.update(blk_hdr) + #hash1_o = hash1.digest() - #hash2 = hashlib.sha256() - #hash2.update(hash1_o) - #hash2_o = hash2.digest() + #hash2 = hashlib.sha256() + #hash2.update(hash1_o) + #hash2_o = hash2.digest() - #return hash2_o + #return hash2_o pow_hash = dash_hash.getPoWHash(blk_hdr) return pow_hash def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hexlify(hash).decode('utf-8') - return hash_str + hash = calc_hdr_hash(blk_hdr) + hash = bufreverse(hash) + hash = wordreverse(hash) + hash_str = hexlify(hash).decode('utf-8') + return hash_str def get_blk_dt(blk_hdr): - members = struct.unpack(" self.maxOutSz): - self.outF.close() - if self.setFileTime: - os.utime(self.outFname, (int(time.time()), self.highTS)) - self.outF = None - self.outFname = None - self.outFn = self.outFn + 1 - self.outsz = 0 + def writeBlock(self, inhdr, blk_hdr, rawblock): + blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) + if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 - (blkDate, blkTS) = get_blk_dt(blk_hdr) - if self.timestampSplit and (blkDate > self.lastDate): - print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) - self.lastDate = blkDate - if self.outF: - self.outF.close() - if self.setFileTime: - os.utime(self.outFname, (int(time.time()), self.highTS)) - self.outF = None - self.outFname = None - self.outFn = self.outFn + 1 - self.outsz = 0 + (blkDate, blkTS) = get_blk_dt(blk_hdr) + if self.timestampSplit and (blkDate > self.lastDate): + print("New month " + blkDate.strftime("%Y-%m") + " @ " + self.hash_str) + self.lastDate = blkDate + if self.outF: + self.outF.close() + if self.setFileTime: + os.utime(self.outFname, (int(time.time()), self.highTS)) + self.outF = None + self.outFname = None + self.outFn = self.outFn + 1 + self.outsz = 0 - if not self.outF: - if self.fileOutput: - self.outFname = self.settings['output_file'] - else: - self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) - print("Output file " + self.outFname) - self.outF = open(self.outFname, "wb") + if not self.outF: + if self.fileOutput: + self.outFname = self.settings['output_file'] + else: + self.outFname = os.path.join(self.settings['output'], "blk%05d.dat" % self.outFn) + print("Output file " + self.outFname) + self.outF = open(self.outFname, "wb") - self.outF.write(inhdr) - self.outF.write(blk_hdr) - self.outF.write(rawblock) - self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) + self.outF.write(inhdr) + self.outF.write(blk_hdr) + self.outF.write(rawblock) + self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) - self.blkCountOut = self.blkCountOut + 1 - if blkTS > self.highTS: - self.highTS = blkTS + self.blkCountOut = self.blkCountOut + 1 + if blkTS > self.highTS: + self.highTS = blkTS - if (self.blkCountOut % 1000) == 0: - print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % - (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) + if (self.blkCountOut % 1000) == 0: + print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % + (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) - def inFileName(self, fn): - return os.path.join(self.settings['input'], "blk%05d.dat" % fn) + def inFileName(self, fn): + return os.path.join(self.settings['input'], "blk%05d.dat" % fn) - def fetchBlock(self, extent): - '''Fetch block contents from disk given extents''' - with open(self.inFileName(extent.fn), "rb") as f: - f.seek(extent.offset) - return f.read(extent.size) + def fetchBlock(self, extent): + '''Fetch block contents from disk given extents''' + with open(self.inFileName(extent.fn), "rb") as f: + f.seek(extent.offset) + return f.read(extent.size) - def copyOneBlock(self): - '''Find the next block to be written in the input, and copy it to the output.''' - extent = self.blockExtents.pop(self.blkCountOut) - if self.blkCountOut in self.outOfOrderData: - # If the data is cached, use it from memory and remove from the cache - rawblock = self.outOfOrderData.pop(self.blkCountOut) - self.outOfOrderSize -= len(rawblock) - else: # Otherwise look up data on disk - rawblock = self.fetchBlock(extent) + def copyOneBlock(self): + '''Find the next block to be written in the input, and copy it to the output.''' + extent = self.blockExtents.pop(self.blkCountOut) + if self.blkCountOut in self.outOfOrderData: + # If the data is cached, use it from memory and remove from the cache + rawblock = self.outOfOrderData.pop(self.blkCountOut) + self.outOfOrderSize -= len(rawblock) + else: # Otherwise look up data on disk + rawblock = self.fetchBlock(extent) - self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) + self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) - def run(self): - while self.blkCountOut < len(self.blkindex): - if not self.inF: - fname = self.inFileName(self.inFn) - print("Input file " + fname) - try: - self.inF = open(fname, "rb") - except IOError: - print("Premature end of block data") - return + def run(self): + while self.blkCountOut < len(self.blkindex): + if not self.inF: + fname = self.inFileName(self.inFn) + print("Input file " + fname) + try: + self.inF = open(fname, "rb") + except IOError: + print("Premature end of block data") + return - inhdr = self.inF.read(8) - if (not inhdr or (inhdr[0] == "\0")): - self.inF.close() - self.inF = None - self.inFn = self.inFn + 1 - continue + inhdr = self.inF.read(8) + if (not inhdr or (inhdr[0] == "\0")): + self.inF.close() + self.inF = None + self.inFn = self.inFn + 1 + continue - inMagic = inhdr[:4] - if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) - return - inLenLE = inhdr[4:] - su = struct.unpack("