From e4978004d0eb4fc2302b6776011f2d49f389fa48 Mon Sep 17 00:00:00 2001 From: grossmj Date: Fri, 17 Jul 2020 21:37:25 +0930 Subject: [PATCH] Update IOUtools. Ref #1627 --- gns3server/compute/iou/utils/iou_export.py | 221 +++++++++++---------- gns3server/compute/iou/utils/iou_import.py | 140 ++++++------- 2 files changed, 172 insertions(+), 189 deletions(-) diff --git a/gns3server/compute/iou/utils/iou_export.py b/gns3server/compute/iou/utils/iou_export.py index 33cdc016..317b6e94 100644 --- a/gns3server/compute/iou/utils/iou_export.py +++ b/gns3server/compute/iou/utils/iou_export.py @@ -37,19 +37,19 @@ optional arguments: -h, --help show this help message and exit """ -import argparse -import sys +import struct -# Uncompress data in .Z file format. -# Ported from dynamips' fs_nvram.c to python -# Adapted from 7zip's ZDecoder.cpp, which is licensed under LGPL 2.1. +# Uncompress data in LZC format, .Z file format +# LZC uses the LZW compression algorithm with a variable dictionary size +# For LZW see https://en.wikipedia.org/wiki/Lempel–Ziv–Welch +# Performance: about 1 MByte/sec, 15-50 times slower than C implementation def uncompress_LZC(data): LZC_NUM_BITS_MIN = 9 LZC_NUM_BITS_MAX = 16 - in_data = bytearray(data) - in_len = len(in_data) + in_data = bytearray(data) + in_len = len(in_data) out_data = bytearray() if in_len == 0: @@ -59,143 +59,144 @@ def uncompress_LZC(data): if in_data[0] != 0x1F or in_data[1] != 0x9D: raise ValueError('invalid header') - maxbits = in_data[2] & 0x1F - numItems = 1 << maxbits - blockMode = (in_data[2] & 0x80) != 0 - if maxbits < LZC_NUM_BITS_MIN or maxbits > LZC_NUM_BITS_MAX: + max_bits = in_data[2] & 0x1F + if max_bits < LZC_NUM_BITS_MIN or max_bits > LZC_NUM_BITS_MAX: raise ValueError('not supported') - - parents = [0] * numItems - suffixes = [0] * numItems - - in_pos = 3 - numBits = LZC_NUM_BITS_MIN - head = 256 - if blockMode: + num_items = 1 << max_bits + blockmode = (in_data[2] & 0x80) != 0 + + in_pos = 3 + start_pos = in_pos + num_bits = LZC_NUM_BITS_MIN + dict_size = 1 << num_bits + head = 256 + if blockmode: head += 1 + first_sym = True + + # initialize dictionary + comp_dict = [None] * num_items + for i in range(0, 256): + comp_dict[i] = bytes(bytearray([i])) - needPrev = 0 - bitPos = 0 - numBufBits = 0 - - parents[256] = 0 - suffixes[256] = 0 - - buf_extend = bytearray([0] * 3) - - while True: - # fill buffer, when empty - if numBufBits == bitPos: - buf_len = min(in_len - in_pos, numBits) - buf = in_data[in_pos:in_pos + buf_len] + buf_extend - numBufBits = buf_len << 3 - bitPos = 0 - in_pos += buf_len - - # extract next symbol - bytePos = bitPos >> 3 - symbol = buf[bytePos] | buf[bytePos + 1] << 8 | buf[bytePos + 2] << 16 - symbol >>= bitPos & 7 - symbol &= (1 << numBits) - 1 - bitPos += numBits - - # check for special conditions: end, bad data, re-initialize dictionary - if bitPos > numBufBits: - break - if symbol >= head: + buf = buf_bits = 0 + while in_pos < in_len: + # get next symbol + try: + while buf_bits < num_bits: + buf |= in_data[in_pos] << buf_bits + buf_bits += 8 + in_pos += 1 + buf, symbol = divmod(buf, dict_size) + buf_bits -= num_bits + except IndexError: raise ValueError('invalid data') - if blockMode and symbol == 256: - numBufBits = bitPos = 0 - numBits = LZC_NUM_BITS_MIN + + # re-initialize dictionary + if blockmode and symbol == 256: + # skip to next buffer boundary + buf = buf_bits = 0 + in_pos += (start_pos - in_pos) % num_bits + # reset to LZC_NUM_BITS_MIN head = 257 - needPrev = 0 + num_bits = LZC_NUM_BITS_MIN + dict_size = 1 << num_bits + start_pos = in_pos + first_sym = True continue - # convert symbol to string - stack = [] - cur = symbol - while cur >= 256: - stack.append(suffixes[cur]) - cur = parents[cur] - stack.append(cur) - if needPrev: - suffixes[head - 1] = cur - if symbol == head - 1: - stack[0] = cur - stack.reverse() - out_data.extend(stack) - - # update parents, check for numBits change - if head < numItems: - needPrev = 1 - parents[head] = symbol - head += 1 - if head > (1 << numBits): - if numBits < maxbits: - numBufBits = bitPos = 0 - numBits += 1 - else: - needPrev = 0 + # first symbol + if first_sym: + first_sym = False + if symbol >= 256: + raise ValueError('invalid data') + prev = symbol + out_data.extend(comp_dict[symbol]) + continue - return out_data + # dictionary full + if head >= num_items: + out_data.extend(comp_dict[symbol]) + continue + # update compression dictionary + if symbol < head: + comp_dict[head] = comp_dict[prev] + comp_dict[symbol][0:1] + elif symbol == head: + comp_dict[head] = comp_dict[prev] + comp_dict[prev][0:1] + else: + raise ValueError('invalid data') + prev = symbol -# extract 16 bit unsigned int from data -def get_uint16(data, off): - return data[off] << 8 | data[off + 1] + # output symbol + out_data.extend(comp_dict[symbol]) + # update head, check for num_bits change + head += 1 + if head >= dict_size and num_bits < max_bits: + num_bits += 1 + dict_size = 1 << num_bits + start_pos = in_pos -# extract 32 bit unsigned int from data -def get_uint32(data, off): - return data[off] << 24 | data[off + 1] << 16 | data[off + 2] << 8 | data[off + 3] + return out_data # export IOU NVRAM +# NVRAM format: https://github.com/ehlers/IOUtools/blob/master/NVRAM.md def nvram_export(nvram): nvram = bytearray(nvram) - # extract startup config offset = 0 - if len(nvram) < offset + 36: - raise ValueError('invalid length') - if get_uint16(nvram, offset + 0) != 0xABCD: - raise ValueError('no startup config') - format = get_uint16(nvram, offset + 2) - length = get_uint32(nvram, offset + 16) - offset += 36 - if len(nvram) < offset + length: + # extract startup config + try: + (magic, data_format, _, _, _, _, length, _, _, _, _, _) = \ + struct.unpack_from('>HHHHIIIIIHHI', nvram, offset=offset) + offset += 36 + if magic != 0xABCD: + raise ValueError('no startup config') + if len(nvram) < offset+length: + raise ValueError('invalid length') + startup = nvram[offset:offset+length] + except struct.error: raise ValueError('invalid length') - startup = nvram[offset:offset + length] - # compressed startup config - if format == 2: + # uncompress startup config + if data_format == 2: try: startup = uncompress_LZC(startup) except ValueError as err: raise ValueError('uncompress startup: ' + str(err)) - offset += length - # alignment to multiple of 4 - offset = (offset + 3) & ~3 - # check for additonal offset of 4 - if len(nvram) >= offset + 8 and \ - get_uint16(nvram, offset + 4) == 0xFEDC and \ - get_uint16(nvram, offset + 6) == 1: - offset += 4 - - # extract private config private = None - if len(nvram) >= offset + 16 and get_uint16(nvram, offset + 0) == 0xFEDC: - length = get_uint32(nvram, offset + 12) + try: + # calculate offset of private header + length += (4 - length % 4) % 4 # alignment to multiple of 4 + offset += length + # check for additonal offset of 4 + (magic, data_format) = struct.unpack_from('>HH', nvram, offset=offset+4) + if magic == 0xFEDC and data_format == 1: + offset += 4 + + # extract private config + (magic, data_format, _, _, length) = \ + struct.unpack_from('>HHIII', nvram, offset=offset) offset += 16 - if len(nvram) >= offset + length: - private = nvram[offset:offset + length] + if magic == 0xFEDC and data_format == 1: + if len(nvram) < offset+length: + raise ValueError('invalid length') + private = nvram[offset:offset+length] + + # missing private header is not an error + except struct.error: + pass return (startup, private) if __name__ == '__main__': # Main program + import argparse + import sys parser = argparse.ArgumentParser(description='%(prog)s exports startup/private configuration from IOU NVRAM file.') parser.add_argument('nvram', metavar='NVRAM', diff --git a/gns3server/compute/iou/utils/iou_import.py b/gns3server/compute/iou/utils/iou_import.py index bd688861..3edad939 100644 --- a/gns3server/compute/iou/utils/iou_import.py +++ b/gns3server/compute/iou/utils/iou_import.py @@ -35,100 +35,82 @@ optional arguments: create NVRAM file, size in kByte """ -import argparse -import sys - - -# extract 16 bit unsigned int from data -def get_uint16(data, off): - return data[off] << 8 | data[off + 1] - - -# extract 32 bit unsigned int from data -def get_uint32(data, off): - return data[off] << 24 | data[off + 1] << 16 | data[off + 2] << 8 | data[off + 3] - - -# insert 16 bit unsigned int into data -def put_uint16(data, off, value): - data[off] = (value >> 8) & 0xff - data[off + 1] = value & 0xff - - -# insert 32 bit unsigned int into data -def put_uint32(data, off, value): - data[off] = (value >> 24) & 0xff - data[off + 1] = (value >> 16) & 0xff - data[off + 2] = (value >> 8) & 0xff - data[off + 3] = value & 0xff +import struct # calculate padding -def padding(off, ios, nvram_len): - pad = (4 - off % 4) % 4 # padding to alignment of 4 - # add 4 if IOS <= 15.0 or NVRAM area >= 64KB - if (ios <= 0x0F00 or nvram_len >= 64 * 1024) and pad != 0: +def padding(length, start_address): + pad = -length % 4 # padding to alignment of 4 + # extra padding if pad != 0 and big start_address + if pad != 0 and (start_address & 0x80000000) != 0: pad += 4 return pad # update checksum def checksum(data, start, end): - put_uint16(data, start + 4, 0) # set checksum to 0 - chk = 0 - idx = start - while idx < end - 1: - chk += get_uint16(data, idx) - idx += 2 - if idx < end: - chk += data[idx] << 8 + # calculate checksum of first two words + for word in struct.unpack_from('>2H', data, start): + chk += word + # add remaining words, ignoring old checksum at offset 4 + struct_format = '>{:d}H'.format((end - start - 6) // 2) + for word in struct.unpack_from(struct_format, data, start+6): + chk += word + + # handle 16 bit overflow while chk >> 16: chk = (chk & 0xffff) + (chk >> 16) - chk = chk ^ 0xffff - put_uint16(data, start + 4, chk) # set checksum + + # save checksum + struct.pack_into('>H', data, start+4, chk) # import IOU NVRAM +# NVRAM format: https://github.com/ehlers/IOUtools/blob/master/NVRAM.md def nvram_import(nvram, startup, private, size): - BASE_ADDRESS = 0x10000000 - DEFAULT_IOS = 0x0F04 # IOS 15.4 + DEFAULT_IOS = 0x0F04 # IOS 15.4 + base_address = 0x10000000 # check size parameter if size is not None and (size < 8 or size > 1024): raise ValueError('invalid size') # create new nvram if nvram is empty or has wrong size - if nvram is None or (size is not None and len(nvram) != size * 1024): - nvram = bytearray([0] * (size * 1024)) + if nvram is None or (size is not None and len(nvram) != size*1024): + nvram = bytearray([0] * (size*1024)) else: nvram = bytearray(nvram) # check nvram size nvram_len = len(nvram) - if nvram_len < 8 * 1024 or nvram_len > 1024 * 1024 or nvram_len % 1024 != 0: + if nvram_len < 8*1024 or nvram_len > 1024*1024 or nvram_len % 1024 != 0: raise ValueError('invalid NVRAM length') nvram_len = nvram_len // 2 # get size of current config config_len = 0 - ios = None try: - if get_uint16(nvram, 0) == 0xABCD: - ios = get_uint16(nvram, 6) - config_len = 36 + get_uint32(nvram, 16) - config_len += padding(config_len, ios, nvram_len) - if get_uint16(nvram, config_len) == 0xFEDC: - config_len += 16 + get_uint32(nvram, config_len + 12) - except IndexError: + (magic, _, _, ios, start_addr, _, length, _, _, _, _, _) = \ + struct.unpack_from('>HHHHIIIIIHHI', nvram, offset=0) + if magic == 0xABCD: + base_address = start_addr - 36 + config_len = 36 + length + padding(length, base_address) + (magic, _, _, _, length) = \ + struct.unpack_from('>HHIII', nvram, offset=config_len) + if magic == 0xFEDC: + config_len += 16 + length + else: + ios = None + except struct.error: raise ValueError('unknown nvram format') if config_len > nvram_len: raise ValueError('unknown nvram format') # calculate max. config size - max_config = nvram_len - 2 * 1024 # reserve 2k for files + max_config = nvram_len - 2*1024 # reserve 2k for files idx = max_config empty_sector = bytearray([0] * 1024) while True: @@ -136,45 +118,43 @@ def nvram_import(nvram, startup, private, size): if idx < config_len: break # if valid file header: - if get_uint16(nvram, idx + 0) == 0xDCBA and \ - get_uint16(nvram, idx + 4) < 8 and \ - get_uint16(nvram, idx + 6) <= 992: + (magic, _, flags, length, _) = \ + struct.unpack_from('>HHHH24s', nvram, offset=idx) + if magic == 0xDCBA and flags < 8 and length <= 992: max_config = idx - elif nvram[idx:idx + 1024] != empty_sector: + elif nvram[idx:idx+1024] != empty_sector: break # import startup config - startup = bytearray(startup) + new_nvram = bytearray() if ios is None: # Target IOS version is unknown. As some IOU don't work nicely with # the padding of a different version, the startup config is padded # with '\n' to the alignment of 4. ios = DEFAULT_IOS - startup.extend([ord('\n')] * ((4 - len(startup) % 4) % 4)) - new_nvram = bytearray([0] * 36) # startup hdr - put_uint16(new_nvram, 0, 0xABCD) # magic - put_uint16(new_nvram, 2, 1) # raw data - put_uint16(new_nvram, 6, ios) # IOS version - put_uint32(new_nvram, 8, BASE_ADDRESS + 36) # start address - put_uint32(new_nvram, 12, BASE_ADDRESS + 36 + len(startup)) # end address - put_uint32(new_nvram, 16, len(startup)) # length + startup += b'\n' * (-len(startup) % 4) + new_nvram.extend(struct.pack('>HHHHIIIIIHHI', + 0xABCD, # magic + 1, # raw data + 0, # checksum, not yet calculated + ios, # IOS version + base_address + 36, # start address + base_address + 36 + len(startup), # end address + len(startup), # length + 0, 0, 0, 0, 0)) new_nvram.extend(startup) - new_nvram.extend([0] * padding(len(new_nvram), ios, nvram_len)) + new_nvram.extend([0] * padding(len(new_nvram), base_address)) # import private config if private is None: - private = bytearray() - else: - private = bytearray(private) + private = b'' offset = len(new_nvram) - new_nvram.extend([0] * 16) # private hdr - put_uint16(new_nvram, 0 + offset, 0xFEDC) # magic - put_uint16(new_nvram, 2 + offset, 1) # raw data - put_uint32(new_nvram, 4 + offset, - BASE_ADDRESS + offset + 16) # start address - put_uint32(new_nvram, 8 + offset, - BASE_ADDRESS + offset + 16 + len(private)) # end address - put_uint32(new_nvram, 12 + offset, len(private)) # length + new_nvram.extend(struct.pack('>HHIII', + 0xFEDC, # magic + 1, # raw data + base_address + offset + 16, # start address + base_address + offset + 16 + len(private), # end address + len(private) )) # length new_nvram.extend(private) # add rest @@ -190,6 +170,8 @@ def nvram_import(nvram, startup, private, size): if __name__ == '__main__': # Main program + import argparse + import sys def check_size(string): try: