fully unpacker rework. Now the unpacker can get the kernel directly from *.npk file!

.gitignore

@@ -2,3 +2,6 @@
 !.gitignore
 kernel.headers
 bins/*
+finder/kernel-extractor
+finder/finder
+ros/*

finder/Build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+gcc ./kernel-extractor.c -o ./kernel-extractor

finder/finder.c

@@ -0,0 +1,56 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+/* kernel-XXX.elf получаемое из npk файла напрямую не может быть распакован. xz ругается на повреждение данных!
+	 сравнительный анализ показал что в данных действительно содержится мусор: [ 7F FF 80 00 00 ]
+	 и так каждые 0x8000 байт.
+
+	 kernel-XXX.elf that obtained from an npk file cannot be directly decompressed. xz reports about data corruption!
+   A comparative analysis showed that the data does contain the garbage: [7F FF 80 00 00]
+   and so every 0x8000 bytes.
+*/
+
+void die(int code){
+	fprintf(stderr, "Dieing with exit code = %d\n", code);
+	exit(code);
+}
+
+static unsigned char data[64 * 1024 * 1024];
+int main(int argc, char *argv[]){
+	int fd;
+	unsigned char *p;
+	void *last_p;
+	int count = 0;
+	size_t len;
+	if(argc !=2)
+		die(-1);
+	fd = open(argv[1], O_RDONLY);
+	if(fd < 0)
+		die(-2);
+	memset(data, 0x0, sizeof(data));
+	len = read(fd, data, sizeof(data));
+	close(fd);
+	if(len <= 0)
+		die(-3);
+	printf("Successfully readed %zd bytes\n", len);
+	printf("Begin of search...\n");
+	last_p = (void*)data;
+	p = (void*)data;
+	p += 0x2F4D;
+	do{
+		if(p[4] == 0x7F && p[3] == 0xFF && p[2] == 0x80 && p[1] == 0x00 && p[0] == 0x00){
+		//if(1){
+			printf("%04d: 0x%08lx: %lu: %02X %02X %02X %02X %02X\n", count++,
+				((void*)p - (void*)data),
+				(void*)p - last_p, p[4], p[3], p[2], p[1], p[0]);
+			last_p = p;
+		}
+		p = (void *)p + 1;
+	}while((void*)p + sizeof(data[0]) - (void*)data < len);
+	printf("...search is Ended\n");
+	return 0;
+}

finder/kernel-extractor.c

@@ -0,0 +1,101 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+void die(int code){
+	fprintf(stderr, "Dieing with exit code = %d\n", code);
+	exit(code);
+}
+
+static unsigned char data[64 * 1024 * 1024];
+
+/* kernel header is 20 bytes:
+	   kernel start magic(8 bytes) - 07 00 01 00 00 00 0A 04
+	   kernel size(4 bytes) - variable
+	   tail magic(3 bytes) - 00 78 01
+	   chunks header or splitter(5 bytes) - 00 00 80 FF 7F or something else(for last chunk)
+*/
+void *do_kernel_start_search(unsigned char *p, void *end){
+	uint64_t *k;
+	do{
+		k = (void*)p;
+		if(*k == 0x040A000000010007LLU){ //check for kernel start magic value
+			k = (void *)k + 8 + 4; //to tail magic
+			/* printf("0x%08lx: %02X %02X %02X, %016lX\n", ((void*)p - (void*)data),
+				p[12], p[13], p[14], *k); */
+			if(*k == 0x7FFF800000017800LL){ //check for tail and chunk splitter values
+				return (void*)p + 15; //return pointer to first chinks splitter
+			}
+		}
+		p = (void *)p + 1;
+	}while((void*)p + 20 <= end);
+	return NULL;
+}
+
+void *do_kernel_extract(unsigned char *p, void *end, char *file_name){
+	int fd;
+	fd = open(file_name, O_WRONLY | O_CREAT);
+	size_t len;
+	int count = 0;
+	if(fd < 0)
+		die(-11);
+	do{
+		//usually chunks are 32768 bytes in size
+		if(p[4] == 0x7F && p[3] == 0xFF && p[2] == 0x80 && p[1] == 0x00 && p[0] == 0x00){
+			//printf("%02X %02X %02X %02X %02X\n", p[4], p[3], p[2], p[1], p[0]);
+			p += 5; //skip chinks splitter
+			len = write(fd, p, 0x8000);
+			//printf("writing chunk %d, len = %zd bytes\n", count, len);
+			count++;
+			if(len != 0x8000){
+				fprintf(stderr, "write to result file '%s' len < 0x8000: %zd\n", file_name, len);
+				break;
+			}
+			p += len;
+		}else{ //but the last chunk may be less
+			//fprintf(stderr, "UNKNOWN tail is: %zd bytes !!!\n", end - (void*)p);
+			printf("%02X %02X %02X %02X %02X\n", p[4], p[3], p[2], p[1], p[0]);
+			len = p[2] << 8 | p[1]; //try to extract current chunk len from chunk header
+			p += 5; //skip chinks splitter
+			if(len > end - (void*)p)
+				len = end - (void*)p;
+			//last chunk?
+			len = write(fd, p, len);
+			printf("last chunk len: %zd\n", len);
+			count++;
+			break;
+		}
+	}while((void*)p + 5 <= end);
+	printf("Successfully writed %d chunks to '%s'\n", count, file_name);
+	close(fd);
+}
+
+int main(int argc, char *argv[]){
+	int fd;
+	void *start;
+	size_t len;
+	if(argc !=3)
+		die(-1);
+	fd = open(argv[1], O_RDONLY);
+	if(fd < 0)
+		die(-2);
+	memset(data, 0x0, sizeof(data));
+	len = read(fd, data, sizeof(data));
+	close(fd);
+	if(len <= 0)
+		die(-3);
+	printf("Successfully readed %zd bytes\n", len);
+	printf("Begin of search...\n");
+	start = do_kernel_start_search(data, (void*)data + len);
+	if(!start){
+		fprintf(stderr, "Can't find kernel start magic(64 bits)\n");
+		die(-4);
+	}
+	printf("...search is Ended\n");
+	printf("kernel start offset is fount at: 0x%08lx\n", start - (void*)data);
+	do_kernel_extract(start, (void*)data + len, argv[2]);
+	return 0;
+}

kernel.elf

@@ -1 +0,0 @@
-./kernel-6.44.elf

unpack-kernel.sh

@@ -0,0 +1,102 @@
+#!/bin/sh
+#
+#(C) Sergey Sergeev aka adron, 2019
+#
+
+_kernel_bin_binwalk=""
+kernel_bin_binwalk(){
+	[ -z "${_kernel_bin_binwalk}" ] && \
+		_kernel_bin_binwalk=$(binwalk ./bins/kernel.bin)
+	echo "${_kernel_bin_binwalk}"
+}
+
+get_xz_offsets(){
+	kernel_bin_binwalk | grep "xz compressed data" | sed -n 's/^\([0-9]\+\).*/\1/p'
+}
+
+get_file_size(){
+	[ -f ${1} ] || {
+		echo 0
+		return 0
+	}
+	du -b ${1} | sed -e 's/\([0-9]\+\).*/\1/'
+}
+
+unpack_kernel_bin(){
+	local xz_offsets
+	local p_count=0
+	local pred_offset
+	local size
+	local offset
+	xz_offsets=$(get_xz_offsets)
+	for offset in ${xz_offsets} 0; do
+		#echo $offset
+		[ -n "${pred_offset}" ] && {
+			p_count=$((${p_count}+1))
+			size=$((${offset}-${pred_offset}))
+			[ ${offset} -eq 0 ] && size=104857600
+			echo "$p_count - ${pred_offset} - ${size}"
+			dd if=./bins/kernel.bin of=./bins/kernel.p${p_count}.xz bs=1 skip=${pred_offset} count=${size}
+		}
+		pred_offset=${offset}
+	done
+}
+
+extract_kernel_elf(){
+	local offsets
+	local offset
+	local kernel_elf_size=0
+	offsets=`kernel_bin_binwalk | sed -n 's/ELF,//p' | sed -n 's/^\([0-9]\+\).*/\1/p'`
+	local elf_last_offset
+	local first_xz_offset
+	for offset in ${offsets}; do
+		elf_last_offset=${offset}
+	done
+	[ -n "${elf_last_offset}" ] && {
+		offsets=$(get_xz_offsets)
+		for offset in ${offsets}; do
+			first_xz_offset=${offset}
+			break
+		done
+		#elf size without xz
+		kernel_elf_size=$((${first_xz_offset}-${elf_last_offset}))
+		local p
+		local size
+		for p in $(seq 1 3); do
+			size=$(get_file_size ./bins/kernel.p${p}.xz)
+			kernel_elf_size=$((${kernel_elf_size}+${size}))
+		done
+		dd if=./bins/kernel.bin bs=1 skip=${elf_last_offset} count=${kernel_elf_size} of=./bins/kernel.elf
+		[ -f ./bins/p3-garbage.bin ] && {
+			offset=484 #[ 45 00 00 00  00 00 00 00  00 00 00 00  01 00 00 00 ] <-- ${offset}
+			dd if=./bins/p3-garbage.bin bs=${offset} count=1 >> ./bins/kernel.elf
+		}
+	}
+}
+
+truncate_kernel_p3(){
+	local garbage_size
+	local p3_size
+	garbage_size=$(get_file_size ./bins/p3-garbage.bin)
+	p3_size=$(get_file_size ./bins/kernel.p3.xz)
+	dd if=./bins/kernel.p3.xz of=./bins/kernel.p3-stripped.xz bs=$((${p3_size}-${garbage_size})) count=1
+	cat ./bins/kernel.p3-stripped.xz > ./bins/kernel.p3.xz
+	rm ./bins/kernel.p3-stripped.xz
+}
+
+unpack_kernel_bin
+
+( xz -dc --single-stream > ./bins/initramfs.cpio && cat > ./bins/p3-garbage.bin ) < ./bins/kernel.p3.xz
+#do kerne.p3.xz fix(cut garbage)
+[ -f ./bins/p3-garbage.bin ] && {
+	truncate_kernel_p3
+}
+
+extract_kernel_elf
+
+exit 0
+
+rm -Rf cpio-fs
+mkdir cpio-fs
+cd cpio-fs
+cpio -idv < ../bins/initramfs.cpio

unpack-npk.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+TARGET=./ros/routeros-arm-6.44.npk
+#TARGET=./ros/routeros-7.0beta1-arm.npk
+FS_BEGIN_OFFSET=4096
+
+get_squashfs_size(){
+	binwalk ${TARGET} | grep "Squashfs filesystem" | sed -n 's/.* size: \([0-9]\+\) bytes.*/\1/p'
+}
+
+FS_SIZE=$(get_squashfs_size)
+[ -z "${FS_SIZE}" ] && {
+	echo "Can't find Squashfs filesystem! in ${TARGET}"
+	exit 1
+}
+
+rm -f ./bins/*
+
+dd if=${TARGET} of=./bins/rootfs.squashfs bs=1 skip=${FS_BEGIN_OFFSET} count=${FS_SIZE}
+#dd if=${TARGET} of=./tail.bin bs=$((${FS_BEGIN_OFFSET}+${FS_SIZE})) skip=1
+./finder/kernel-extractor ${TARGET} ./bins/kernel.bin
+
+exit 0

unpack.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-#
-#(C) Sergey Sergeev aka adron, 2019
-#
-
-#exit 0
-
-#for 6.43 kernel
-#dd if=kernel.elf bs=1 skip=0 count=43504 of=./bins/kernel.HEAD
-#dd if=kernel.elf bs=1 skip=43504 count=1605632 of=./bins/kernel.p2.xz
-#dd if=kernel.elf bs=1 skip=1649136 count=84812 of=./bins/kernel.p3.xz
-#dd if=kernel.elf bs=1 skip=1733948 of=./bins/kernel.TAIL
-
-#for 6.44 kernel
-dd if=kernel.elf bs=1 skip=43752 count=1609728 of=./bins/kernel.p2.xz
-dd if=kernel.elf bs=1 skip=1653480 count=85128 of=./bins/kernel.p3.xz
-
-
-#( xz -dc --single-stream > ./bins/initramfs.cpio && cat > ./bins/p3-tail.bin ) < ./bins/kernel.p3.xz
-xz -dc > ./bins/initramfs.cpio < ./bins/kernel.p3.xz
-
-exit 0
-
-rm -Rf cpio-fs
-mkdir cpio-fs
-cd cpio-fs
-cpio -idv < ../bins/initramfs.cpio