Initial commit
This commit is contained in:
commit
796c991c85
14 changed files with 911 additions and 0 deletions
7
LICENSE
Normal file
7
LICENSE
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
Copyright © 2025 John Lorentzson (Duuqnd)
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
26
Makefile
Normal file
26
Makefile
Normal file
|
@ -0,0 +1,26 @@
|
||||||
|
|
||||||
|
|
||||||
|
BUILD_DIR = ./build
|
||||||
|
SRC_DIR = ./src
|
||||||
|
|
||||||
|
SRCS = $(shell find $(SRC_DIR) -name '*.c')
|
||||||
|
OBJS = $(SRCS:%=$(BUILD_DIR)/%.o)
|
||||||
|
DEPS = $(OBJS:.o=.d)
|
||||||
|
|
||||||
|
INC_FLAGS := $(addprefix -I,$(SRC_DIR))
|
||||||
|
CPPFLAGS := $(INC_FLAGS) -MMD -MP
|
||||||
|
|
||||||
|
all: bindiff
|
||||||
|
|
||||||
|
bindiff: $(OBJS)
|
||||||
|
$(CC) $(CFLAGS) -o $@ $(OBJS)
|
||||||
|
|
||||||
|
$(BUILD_DIR)/%.c.o: %.c
|
||||||
|
mkdir -p $(dir $@)
|
||||||
|
$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ -c $<
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
|
clean:
|
||||||
|
rm -r $(BUILD_DIR) bindiff
|
||||||
|
|
||||||
|
-include $(DEPS)
|
23
README.md
Normal file
23
README.md
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
# bindiff
|
||||||
|
|
||||||
|
A program for making delta backups of binary files. It works in units of blocks, which by default are 8192 bytes large. A partial snapshot will contain complete copies of only the blocks that have changed since the most recent snapshot. When a partial snapshot would be larger than a complete copy, a complete snapshot is made. Partial snapshots depend on all the snapshots between it and the most recent complete snapshot. This means that if you have a complete snapshot made recently, you can delete all prior snapshots.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
A set of snapshots of one file are stored in one directory. Snapshots of different files cannot share a directory. To create a snapshot, use the following command:
|
||||||
|
|
||||||
|
`bindiff snapshot the-file-to-save snapshot-directory`
|
||||||
|
|
||||||
|
If there are no existing snapshots in `snapshot-directory`, an initial complete snapshot is created. If snapshots do exist, a snapshot of appropriate type will be created. The automatically created file name contains both the file name of the original file, a UNIX timestamp, and a file extension hinting at the snapshot type (`bdo` for original, `bdc` for complete, `bdp` for partial).
|
||||||
|
|
||||||
|
To restore a complete file from a snapshot, use the following command:
|
||||||
|
|
||||||
|
`bindiff restore snapshot-directory snapshot-file file-output`
|
||||||
|
|
||||||
|
The program will start at the timestamp found in snapshot `snapshot-file` and work backward through the snapshots in `snapshot-directory` to reconstruct the entire file, writing this reconstruction to `file-output`.
|
||||||
|
|
||||||
|
To describe a snapshot, use the following command:
|
||||||
|
|
||||||
|
`bindiff describe snapshot-file`
|
||||||
|
|
||||||
|
A list of properties of the snapshot will be printed to the terminal. This format is not guaranteed to be stable.
|
29
src/bitmap.c
Normal file
29
src/bitmap.c
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
#include "bitmap.h"
|
||||||
|
#include "blocks.h"
|
||||||
|
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
uint32_t* changesBitmap;
|
||||||
|
|
||||||
|
void setBit(uint32_t* bitmap, size_t bit) {
|
||||||
|
bitmap[bit / BITS_PER_WORD] |= (1 << (bit % BITS_PER_WORD));
|
||||||
|
}
|
||||||
|
|
||||||
|
void clearBit(uint32_t* bitmap, size_t bit) {
|
||||||
|
bitmap[bit / BITS_PER_WORD] &= ~(1 << (bit % BITS_PER_WORD));
|
||||||
|
}
|
||||||
|
|
||||||
|
uint8_t getBit(uint32_t* bitmap, size_t bit) {
|
||||||
|
return (bitmap[bit / BITS_PER_WORD] & (1 << (bit % BITS_PER_WORD))) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void setupChangesBitmap(size_t blockCount) {
|
||||||
|
size_t length = (blockSize * blockCount) / BITS_PER_WORD;
|
||||||
|
changesBitmap = (uint32_t*)malloc(length);
|
||||||
|
memset(changesBitmap, 0, length);
|
||||||
|
}
|
||||||
|
|
||||||
|
void teardownChangesBitmap() {
|
||||||
|
free(changesBitmap);
|
||||||
|
}
|
14
src/bitmap.h
Normal file
14
src/bitmap.h
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
|
||||||
|
#define BITS_PER_WORD 32
|
||||||
|
|
||||||
|
extern uint32_t* changesBitmap;
|
||||||
|
|
||||||
|
void setBit(uint32_t* bitmap, size_t bit);
|
||||||
|
void clearBit(uint32_t* bitmap, size_t bit);
|
||||||
|
uint8_t getBit(uint32_t* bitmap, size_t bit);
|
||||||
|
void setupChangesBitmap(size_t blockCount);
|
||||||
|
void teardownChangesBitmap();
|
19
src/blocks.c
Normal file
19
src/blocks.c
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
#include "blocks.h"
|
||||||
|
#include "bitmap.h"
|
||||||
|
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
uint32_t blockSize = 8192;
|
||||||
|
|
||||||
|
uint8_t* oldBlock;
|
||||||
|
uint8_t* newBlock;
|
||||||
|
|
||||||
|
void setupBlockStorage() {
|
||||||
|
oldBlock = (uint8_t*)malloc(blockSize);
|
||||||
|
newBlock = (uint8_t*)malloc(blockSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
void teardownBlockStorage() {
|
||||||
|
free(oldBlock);
|
||||||
|
free(newBlock);
|
||||||
|
}
|
11
src/blocks.h
Normal file
11
src/blocks.h
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
extern uint32_t blockSize;
|
||||||
|
|
||||||
|
extern uint8_t* oldBlock;
|
||||||
|
extern uint8_t* newBlock;
|
||||||
|
|
||||||
|
void setupBlockStorage();
|
||||||
|
void teardownBlockStorage();
|
91
src/header.c
Normal file
91
src/header.c
Normal file
|
@ -0,0 +1,91 @@
|
||||||
|
#include "header.h"
|
||||||
|
#include "bitmap.h"
|
||||||
|
#include "blocks.h"
|
||||||
|
#include "snapshot.h"
|
||||||
|
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <sys/time.h>
|
||||||
|
|
||||||
|
char* headerError;
|
||||||
|
|
||||||
|
void writeHeader(FILE* fp, size_t diffLength, uint64_t previousTimestamp) {
|
||||||
|
// Magic, identifies a BlockDiff snapshot
|
||||||
|
uint8_t* magic = "BD";
|
||||||
|
fwrite((void*)magic, 2, sizeof(uint8_t), fp);
|
||||||
|
|
||||||
|
// Endianess indicator, allows the reconstructing process to know whether
|
||||||
|
// or not it needs to byteswap data fields.
|
||||||
|
uint16_t endianCheck = 0xBEEF;
|
||||||
|
fwrite((void*)&endianCheck, 1, sizeof(uint16_t), fp);
|
||||||
|
|
||||||
|
// Block size of the snapshot, self explanatory. Must be the same for all
|
||||||
|
// snapshots in a chain.
|
||||||
|
fwrite((void*)&blockSize, 1, sizeof(uint32_t), fp);
|
||||||
|
|
||||||
|
// The number of blocks that changed in this snapshot. Set to -1 when the
|
||||||
|
// snapshot is a complete disk image and not a diff.
|
||||||
|
uint64_t lengthField = (uint64_t)diffLength;
|
||||||
|
fwrite((void*)&lengthField, 1, sizeof(uint64_t), fp);
|
||||||
|
|
||||||
|
// Current time of the snapshot being made. This also functions as the
|
||||||
|
// snapshot's identity when reconstructing disk image from snapshots.
|
||||||
|
struct timeval now;
|
||||||
|
gettimeofday(&now, NULL);
|
||||||
|
uint64_t timestamp = (uint64_t)now.tv_sec;
|
||||||
|
fwrite((void*)×tamp, 1, sizeof(uint64_t), fp);
|
||||||
|
|
||||||
|
// The timestamp of the disk image this one was diffed from. Set to -1 in
|
||||||
|
// the initial snapshot as it has no previous.
|
||||||
|
fwrite((void*)&previousTimestamp, 1, sizeof(uint64_t), fp);
|
||||||
|
}
|
||||||
|
|
||||||
|
void writeBlockList(FILE* fp, uint64_t totalNumBlocks, uint32_t* bitmap, uint64_t* blockNums) {
|
||||||
|
uint64_t found = 0;
|
||||||
|
for(size_t i = 0; i < totalNumBlocks; i++) {
|
||||||
|
if(getBit(bitmap, i) == 1) {
|
||||||
|
uint64_t blockNum = i;
|
||||||
|
blockNums[found] = blockNum;
|
||||||
|
found += 1;
|
||||||
|
fwrite(&blockNum, 1, sizeof(uint64_t), fp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool readHeader(FILE* fp, uint64_t* numBlocks, uint64_t* timestamp, uint64_t* previousTimestamp) {
|
||||||
|
uint8_t magic[2];
|
||||||
|
fread(magic, 2, sizeof(uint8_t), fp);
|
||||||
|
|
||||||
|
if(magic[0] != 'B' || magic[1] != 'D') {
|
||||||
|
headerError = "Incorrect header magic string. Expected \"BD\".\n";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint16_t endianCheck;
|
||||||
|
fread(&endianCheck, 1, sizeof(uint16_t), fp);
|
||||||
|
|
||||||
|
if(endianCheck == 0xEFBE) {
|
||||||
|
headerError = "Incorrect endian. Not yet supported.\n";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(endianCheck != 0xBEEF) {
|
||||||
|
fprintf(stderr, "Incorrect endian check bytes 0x%04X. Expected 0xBEEF or 0xEFBE.\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Check that this matches what's already there.
|
||||||
|
fread(&blockSize, 1, sizeof(uint32_t), fp);
|
||||||
|
|
||||||
|
fread(numBlocks, 1, sizeof(uint64_t), fp);
|
||||||
|
fread(timestamp, 1, sizeof(uint64_t), fp);
|
||||||
|
fread(previousTimestamp, 1, sizeof(uint64_t), fp);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t preambleSize(size_t blockCount) {
|
||||||
|
size_t header = HEADER_LENGTH;
|
||||||
|
size_t blockList = blockCount * sizeof(uint64_t);
|
||||||
|
|
||||||
|
return header + blockList;
|
||||||
|
}
|
13
src/header.h
Normal file
13
src/header.h
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#define HEADER_LENGTH 32
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
void writeHeader(FILE* fp, size_t diffLength, uint64_t previousTimestamp);
|
||||||
|
void writeBlockList(FILE* fp, uint64_t totalNumBlocks, uint32_t* bitmap, uint64_t* blockNums);
|
||||||
|
bool readHeader(FILE* fp, uint64_t* numBlocks, uint64_t* timestamp, uint64_t* previousTimestamp);
|
||||||
|
size_t preambleSize(size_t blockCount);
|
212
src/make-diff.c
Normal file
212
src/make-diff.c
Normal file
|
@ -0,0 +1,212 @@
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
#include <assert.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/time.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
|
||||||
|
#include "header.h"
|
||||||
|
#include "snapshot.h"
|
||||||
|
#include "blocks.h"
|
||||||
|
#include "bitmap.h"
|
||||||
|
#include "reconstruction.h"
|
||||||
|
|
||||||
|
size_t diffSize(size_t blockCount) {
|
||||||
|
size_t blocks = blockCount * blockSize;
|
||||||
|
return preambleSize(blockCount) + blocks;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t fileLength(FILE* fp) {
|
||||||
|
size_t length = -1;
|
||||||
|
size_t hold = ftell(fp);
|
||||||
|
|
||||||
|
fseek(fp, 0, SEEK_END);
|
||||||
|
length = ftell(fp);
|
||||||
|
fseek(fp, hold, SEEK_SET);
|
||||||
|
|
||||||
|
return length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* if(diskLength != fileLength(newFp)) { */
|
||||||
|
/* fprintf(stderr, "Old and new disk are different sizes.\n"); */
|
||||||
|
/* exit(-1); */
|
||||||
|
/* } */
|
||||||
|
|
||||||
|
|
||||||
|
void newSnapshot(char* snapshotDir, char* basename, FILE* newState) {
|
||||||
|
bool isInitial = false;
|
||||||
|
bool makeComplete;
|
||||||
|
size_t differenceCount;
|
||||||
|
|
||||||
|
size_t diskLength = fileLength(newState);
|
||||||
|
|
||||||
|
if(diskLength % blockSize != 0) {
|
||||||
|
fprintf(stderr, "Disk size %d is not a clean amount of whole %d byte large blocks.\n",
|
||||||
|
diskLength, blockSize);
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// No snapshots exist, create inital starting snapshot
|
||||||
|
if(snapshotsExist(snapshotDir) == false) {
|
||||||
|
// TODO: Verify that the user really means that directory.
|
||||||
|
isInitial = true;
|
||||||
|
makeComplete = true;
|
||||||
|
printf("No previous snapshots exist at the path, creating initial snapshot.\n");
|
||||||
|
} else {
|
||||||
|
// Snapshots exist, create a new one based on differences from the newest
|
||||||
|
loadInSnapshots(snapshotDir, true, 0);
|
||||||
|
|
||||||
|
size_t numBlocks = diskLength / blockSize;
|
||||||
|
|
||||||
|
setupChangesBitmap(numBlocks);
|
||||||
|
setupBlockStorage();
|
||||||
|
|
||||||
|
differenceCount = 0;
|
||||||
|
|
||||||
|
for(size_t i = 0; i < numBlocks; i++) {
|
||||||
|
bool different = isNewBlockDifferent(i, newState);
|
||||||
|
if(different) {
|
||||||
|
differenceCount += 1;
|
||||||
|
setBit(changesBitmap, i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t storedDiffSize = diffSize(differenceCount);
|
||||||
|
|
||||||
|
printf("%d blocks have changed. Diff would be ~%dK large.\n", differenceCount,
|
||||||
|
storedDiffSize / 1024);
|
||||||
|
|
||||||
|
if(storedDiffSize >= diskLength) {
|
||||||
|
makeComplete = true;
|
||||||
|
printf("Larger than disk being diffed, making complete image snapshot.\n");
|
||||||
|
} else {
|
||||||
|
makeComplete = false;
|
||||||
|
printf("Smaller than disk being diffed, making partial snapshot.\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(makeComplete) {
|
||||||
|
writeCompleteSnapshot(snapshotDir, basename, newState, isInitial);
|
||||||
|
} else {
|
||||||
|
writePartialSnapshot(snapshotDir, basename, newState, differenceCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
teardownBlockStorage();
|
||||||
|
teardownChangesBitmap();
|
||||||
|
}
|
||||||
|
|
||||||
|
void restoreSnapshot(char* snapshotDir, char* snapshotFile, FILE* output) {
|
||||||
|
uint64_t timestamp = peekSnapshotFileTimestamp(snapshotFile);
|
||||||
|
loadInSnapshots(snapshotDir, false, timestamp);
|
||||||
|
|
||||||
|
uint8_t block[blockSize];
|
||||||
|
|
||||||
|
for(size_t i; i < snapshotTotalBlocks; i++) {
|
||||||
|
reconstructBlock(i, block);
|
||||||
|
fwrite(block, blockSize, 1, output);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void describeSnapshot(FILE* snapshotFile) {
|
||||||
|
Snapshot* sp = makeSnapshot(snapshotFile);
|
||||||
|
|
||||||
|
printf("Snapshot type: %s\n", isSnapshotComplete(sp) ? "complete" : "partial");
|
||||||
|
printf("Block size: %d\n", blockSize);
|
||||||
|
|
||||||
|
if(isSnapshotComplete(sp) == false) {
|
||||||
|
printf("Changed blocks: %ld\n", sp->blockCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("Creation timestamp: %ld\n", sp->timestamp);
|
||||||
|
if(isSnapshotInitial(sp)) {
|
||||||
|
printf("This snapshot is an initial starting snapshot.\n");
|
||||||
|
} else {
|
||||||
|
printf("Last snapshot's timestamp: %ld\n", sp->previousTimestamp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum RunMode {
|
||||||
|
ERROR,
|
||||||
|
SNAPSHOT,
|
||||||
|
RESTORE,
|
||||||
|
DESCRIBE
|
||||||
|
};
|
||||||
|
|
||||||
|
static void commandFail(char* argv0) {
|
||||||
|
fprintf(stderr, "usage: %s command options, where command is snapshot or restore\n", argv0);
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char* argv[]) {
|
||||||
|
enum RunMode runMode = ERROR;
|
||||||
|
|
||||||
|
if(argc < 2) {
|
||||||
|
commandFail(argv[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(strcmp(argv[1], "snapshot") == 0) {
|
||||||
|
runMode = SNAPSHOT;
|
||||||
|
} else if(strcmp(argv[1], "restore") == 0) {
|
||||||
|
runMode = RESTORE;
|
||||||
|
} else if(strcmp(argv[1], "describe") == 0) {
|
||||||
|
runMode = DESCRIBE;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(runMode == ERROR) {
|
||||||
|
commandFail(argv[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(runMode == SNAPSHOT && argc != 4) {
|
||||||
|
fprintf(stderr, "usage: %s snapshot new-disk-image snapshot-directory\n", argv[0]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(runMode == RESTORE && argc != 5) {
|
||||||
|
fprintf(stderr, "usage: %s restore snapshot-directory snapshot-file restored-disk-image\n", argv[0]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(runMode == DESCRIBE && argc != 3) {
|
||||||
|
fprintf(stderr, "usage: %s describe snapshot-file\n", argv[0]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(runMode == SNAPSHOT) {
|
||||||
|
FILE* newFp = fopen(argv[2], "rb");
|
||||||
|
|
||||||
|
if(newFp == NULL) {
|
||||||
|
fprintf(stderr, "Failed to open %s for reading.\n", argv[2]);
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
newSnapshot(argv[3], argv[2], newFp);
|
||||||
|
|
||||||
|
fclose(newFp);
|
||||||
|
} else if(runMode == RESTORE) {
|
||||||
|
FILE* output = fopen(argv[4], "wb");
|
||||||
|
|
||||||
|
if(output == NULL) {
|
||||||
|
fprintf(stderr, "Failed to open %s for writing.\n", argv[4]);
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
restoreSnapshot(argv[2], argv[3], output);
|
||||||
|
|
||||||
|
fclose(output);
|
||||||
|
} else if(runMode == DESCRIBE) {
|
||||||
|
FILE* fp = fopen(argv[2], "rb");
|
||||||
|
if(fp == NULL) {
|
||||||
|
fprintf(stderr, "Failed to open %s for reading.\n", argv[2]);
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
describeSnapshot(fp);
|
||||||
|
|
||||||
|
fclose(fp);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
48
src/reconstruction.c
Normal file
48
src/reconstruction.c
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
#include "reconstruction.h"
|
||||||
|
#include "blocks.h"
|
||||||
|
#include "snapshot.h"
|
||||||
|
#include "bitmap.h"
|
||||||
|
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
|
void reconstructBlock(uint64_t blockNum, uint8_t* dst) {
|
||||||
|
Snapshot* sp = latestSnapshot;
|
||||||
|
bool found = false;
|
||||||
|
|
||||||
|
while(found == false) {
|
||||||
|
if(isSnapshotComplete(sp)) {
|
||||||
|
completeSnapshotReadBlock(sp, blockNum, dst);
|
||||||
|
found = true;
|
||||||
|
} else if(isSnapshotAtBlock(sp, blockNum)) {
|
||||||
|
snapshotReadBlockAtCursor(sp, dst);
|
||||||
|
found = true;
|
||||||
|
} else if(isSnapshotBlockAfter(sp, blockNum) || isSnapshotDone(sp)) {
|
||||||
|
assert(sp->previous != NULL);
|
||||||
|
sp = sp->previous;
|
||||||
|
} else {
|
||||||
|
snapshotNextBlock(sp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void readNewAndOldBlock(uint64_t blockNum, FILE* new) {
|
||||||
|
fseek(new, blockNum * blockSize, SEEK_SET);
|
||||||
|
fread(newBlock, blockSize, 1, new);
|
||||||
|
|
||||||
|
reconstructBlock(blockNum, oldBlock);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isNewBlockDifferent(uint64_t blockNum, FILE* new) {
|
||||||
|
// Sets the block's bit and returns true if it's changed.
|
||||||
|
// Otherwise returns false.
|
||||||
|
readNewAndOldBlock(blockNum, new);
|
||||||
|
|
||||||
|
for(size_t i = 0; i < blockSize; i++) {
|
||||||
|
if(newBlock[i] != oldBlock[i]) {
|
||||||
|
setBit(changesBitmap, blockNum);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
8
src/reconstruction.h
Normal file
8
src/reconstruction.h
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
void reconstructBlock(uint64_t blockNum, uint8_t* dst);
|
||||||
|
void readNewAndOldBlock(uint64_t blockNum, FILE* new);
|
||||||
|
bool isNewBlockDifferent(uint64_t blockNum, FILE* new);
|
367
src/snapshot.c
Normal file
367
src/snapshot.c
Normal file
|
@ -0,0 +1,367 @@
|
||||||
|
#include "snapshot.h"
|
||||||
|
#include "reconstruction.h"
|
||||||
|
#include "blocks.h"
|
||||||
|
#include "header.h"
|
||||||
|
#include "bitmap.h"
|
||||||
|
|
||||||
|
#include <sys/time.h>
|
||||||
|
|
||||||
|
#include <assert.h>
|
||||||
|
#include <dirent.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
Snapshot* latestSnapshot;
|
||||||
|
uint64_t snapshotTotalBlocks = 0;
|
||||||
|
|
||||||
|
// tmp solution
|
||||||
|
// TODO: move to better place
|
||||||
|
static size_t fileLength(FILE* fp) {
|
||||||
|
size_t length = -1;
|
||||||
|
size_t hold = ftell(fp);
|
||||||
|
|
||||||
|
fseek(fp, 0, SEEK_END);
|
||||||
|
length = ftell(fp);
|
||||||
|
fseek(fp, hold, SEEK_SET);
|
||||||
|
|
||||||
|
return length;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checks on snapshot metadata
|
||||||
|
|
||||||
|
bool isSnapshotComplete(Snapshot* sp) {
|
||||||
|
return sp->blockCount == (uint64_t)-1;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isSnapshotInitial(Snapshot* sp) {
|
||||||
|
return sp->previousTimestamp == (uint64_t)-1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Making snapshot metadata structures
|
||||||
|
|
||||||
|
Snapshot* makeSnapshot(FILE* fp) {
|
||||||
|
Snapshot* sp = (Snapshot*)malloc(sizeof(Snapshot));
|
||||||
|
sp->fp = fp;
|
||||||
|
|
||||||
|
readHeader(sp->fp, &sp->blockCount, &sp->timestamp, &sp->previousTimestamp);
|
||||||
|
|
||||||
|
if(isSnapshotComplete(sp)) {
|
||||||
|
if(snapshotTotalBlocks == 0) {
|
||||||
|
snapshotTotalBlocks = fileLength(sp->fp) / blockSize;
|
||||||
|
} else {
|
||||||
|
assert(snapshotTotalBlocks == sp->blockCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sp->cursor = HEADER_LENGTH;
|
||||||
|
sp->blockListIndex = -1;
|
||||||
|
snapshotNextBlock(sp);
|
||||||
|
|
||||||
|
return sp;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Making snapshot files
|
||||||
|
|
||||||
|
static size_t snapshotFilenameLength(char* snapshotDir, char* basename, char* timestring) {
|
||||||
|
return strlen(snapshotDir) + 1 /* slash */
|
||||||
|
+ strlen(basename) + 1 /* name/time separator */
|
||||||
|
+ strlen(timestring)
|
||||||
|
+ 4 /* dot & file extention */
|
||||||
|
+ 1; /* NULL terminator */
|
||||||
|
}
|
||||||
|
|
||||||
|
static char* makeTimestring(struct timeval* time) {
|
||||||
|
uint64_t timestamp = time->tv_sec;
|
||||||
|
|
||||||
|
// 20 digits is enough for 2^64, + 1 for NULL terminator
|
||||||
|
// most timestrings will be shorter
|
||||||
|
char* timestring = (char*)malloc(20 + 1);
|
||||||
|
memset(timestring, 0, 20 + 1);
|
||||||
|
snprintf(timestring, 20, "%d", timestamp);
|
||||||
|
|
||||||
|
return timestring;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char* makeSnapshotFilename(bool initial, bool partial, char* snapshotDir,
|
||||||
|
char* basename, char* timestring) {
|
||||||
|
char* output = (char*)malloc(snapshotFilenameLength(snapshotDir, basename, timestring));
|
||||||
|
output[0] = 0;
|
||||||
|
|
||||||
|
char* extension;
|
||||||
|
|
||||||
|
if(initial) {
|
||||||
|
assert(partial == false);
|
||||||
|
extension = "bdo";
|
||||||
|
} else if(partial) {
|
||||||
|
extension = "bdp";
|
||||||
|
} else {
|
||||||
|
extension = "bdc";
|
||||||
|
}
|
||||||
|
|
||||||
|
strcat(output, snapshotDir);
|
||||||
|
strcat(output, "/");
|
||||||
|
strcat(output, basename);
|
||||||
|
strcat(output, "_");
|
||||||
|
strcat(output, timestring);
|
||||||
|
strcat(output, ".");
|
||||||
|
strcat(output, extension);
|
||||||
|
strcat(output, "\0");
|
||||||
|
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
static FILE* openNewSnapshotFile(char* snapshotDir, char* basename, bool initial, bool partial) {
|
||||||
|
struct timeval now;
|
||||||
|
gettimeofday(&now, NULL);
|
||||||
|
char* timestring = makeTimestring(&now);
|
||||||
|
char* filepath = makeSnapshotFilename(initial, partial,
|
||||||
|
snapshotDir, basename, timestring);
|
||||||
|
free(timestring);
|
||||||
|
|
||||||
|
// TODO: Error checking
|
||||||
|
FILE* fp = fopen(filepath, "wb");
|
||||||
|
free(filepath);
|
||||||
|
|
||||||
|
return fp;
|
||||||
|
}
|
||||||
|
|
||||||
|
void writeCompleteSnapshot(char* snapshotDir, char* basename, FILE* raw, bool initial) {
|
||||||
|
FILE* output = openNewSnapshotFile(snapshotDir, basename, initial, false);
|
||||||
|
writeHeader(output, -1, initial ? -1 : latestSnapshot->timestamp);
|
||||||
|
|
||||||
|
size_t length = fileLength(raw);
|
||||||
|
for(size_t i = 0; i < length; i++) {
|
||||||
|
uint8_t byte;
|
||||||
|
fread(&byte, 1, sizeof(uint8_t), raw);
|
||||||
|
fwrite(&byte, 1, sizeof(uint8_t), output);
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose(output);
|
||||||
|
}
|
||||||
|
|
||||||
|
void writePartialSnapshot(char* snapshotDir, char* basename, FILE* new, uint64_t differenceCount) {
|
||||||
|
FILE* output = openNewSnapshotFile(snapshotDir, basename, false, true);
|
||||||
|
writeHeader(output, differenceCount, latestSnapshot->timestamp);
|
||||||
|
|
||||||
|
uint64_t blockNums[differenceCount];
|
||||||
|
writeBlockList(output, snapshotTotalBlocks, changesBitmap, blockNums);
|
||||||
|
snapshotRewindAllCursors();
|
||||||
|
|
||||||
|
uint8_t block[blockSize];
|
||||||
|
|
||||||
|
|
||||||
|
for(size_t i = 0; i < differenceCount; i++) {
|
||||||
|
size_t blockOffset = blockNums[i] * blockSize;
|
||||||
|
fseek(new, blockOffset, SEEK_SET);
|
||||||
|
fread(block, blockSize, 1, new);
|
||||||
|
fwrite(block, blockSize, 1, output);
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose(output);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quick peeks into snapshot files
|
||||||
|
|
||||||
|
bool fileIsSnapshot(char* filepath) {
|
||||||
|
FILE* fp = fopen(filepath, "rb");
|
||||||
|
if(fp == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t dummy1, dummy2, dummy3;
|
||||||
|
bool isSnapshot = readHeader(fp, &dummy1, &dummy2, &dummy3);
|
||||||
|
fclose(fp);
|
||||||
|
return isSnapshot;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t peekSnapshotFileTimestamp(char* filepath) {
|
||||||
|
FILE* fp = fopen(filepath, "rb");
|
||||||
|
uint64_t dummy1, timestamp, dummy2;
|
||||||
|
bool isSnapshot = readHeader(fp, &dummy1, ×tamp, &dummy2);
|
||||||
|
assert(isSnapshot == true);
|
||||||
|
fclose(fp);
|
||||||
|
return timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot block reading
|
||||||
|
|
||||||
|
bool isSnapshotAtBlock(Snapshot* sp, uint64_t blockNum) {
|
||||||
|
return blockNum == sp->blockAtCursor;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isSnapshotBlockAfter(Snapshot* sp, uint64_t blockNum) {
|
||||||
|
return blockNum < sp->blockAtCursor;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool isSnapshotDone(Snapshot* sp) {
|
||||||
|
return sp->blockListIndex >= sp->blockCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool snapshotNextBlock(Snapshot* sp) {
|
||||||
|
sp->blockListIndex += 1;
|
||||||
|
if(sp->blockListIndex < sp->blockCount) {
|
||||||
|
fseek(sp->fp, sp->cursor, SEEK_SET);
|
||||||
|
uint64_t blockNum;
|
||||||
|
fread(&blockNum, 1, sizeof(uint64_t), sp->fp);
|
||||||
|
|
||||||
|
sp->blockAtCursor = blockNum;
|
||||||
|
sp->cursor = ftell(sp->fp);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void snapshotRewindCursor(Snapshot* sp) {
|
||||||
|
rewind(sp->fp);
|
||||||
|
sp->cursor = HEADER_LENGTH;
|
||||||
|
sp->blockListIndex = -1;
|
||||||
|
snapshotNextBlock(sp);
|
||||||
|
}
|
||||||
|
|
||||||
|
void snapshotRewindAllCursors() {
|
||||||
|
Snapshot* iter = latestSnapshot;
|
||||||
|
while(iter != NULL) {
|
||||||
|
snapshotRewindCursor(iter);
|
||||||
|
iter = iter->previous;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void snapshotReadBlockAtCursor(Snapshot* sp, uint8_t* dst) {
|
||||||
|
size_t offset = preambleSize(sp->blockCount) +
|
||||||
|
(sp->blockListIndex * blockSize);
|
||||||
|
|
||||||
|
fseek(sp->fp, offset, SEEK_SET);
|
||||||
|
fread(dst, blockSize, 1, sp->fp);
|
||||||
|
}
|
||||||
|
|
||||||
|
void completeSnapshotReadBlock(Snapshot* sp, uint64_t blockNum, uint8_t* dst) {
|
||||||
|
size_t offset = HEADER_LENGTH + (blockNum * blockSize);
|
||||||
|
|
||||||
|
fseek(sp->fp, offset, SEEK_SET);
|
||||||
|
fread(dst, blockSize, 1, sp->fp);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finding and opening existing snapshots
|
||||||
|
|
||||||
|
bool snapshotsExist(char* dirpath) {
|
||||||
|
// TODO: Check that files that exist are BD snapshots
|
||||||
|
bool filesExist = false;
|
||||||
|
DIR* snapshotDir = opendir(dirpath);
|
||||||
|
char pathname[256 * 2];
|
||||||
|
|
||||||
|
struct dirent* entry = readdir(snapshotDir);
|
||||||
|
while(entry != NULL) {
|
||||||
|
memset(pathname, 0, 256 * 2);
|
||||||
|
strcat(pathname, dirpath);
|
||||||
|
strcat(pathname, "/");
|
||||||
|
strcat(pathname, entry->d_name);
|
||||||
|
|
||||||
|
if(fileIsSnapshot(pathname)) {
|
||||||
|
filesExist = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
entry = readdir(snapshotDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
closedir(snapshotDir);
|
||||||
|
|
||||||
|
return filesExist;
|
||||||
|
}
|
||||||
|
|
||||||
|
FILE* openNewestSnapshot(char* snapshotDir) {
|
||||||
|
DIR* dir = opendir(snapshotDir);
|
||||||
|
uint64_t highestTimestamp = 0;
|
||||||
|
char filepath[256 * 2];
|
||||||
|
char newestFilename[256] = { 0 };
|
||||||
|
|
||||||
|
struct dirent* entry = readdir(dir);
|
||||||
|
while(entry != NULL) {
|
||||||
|
memset(filepath, 0, 256 * 2);
|
||||||
|
strcat(filepath, snapshotDir);
|
||||||
|
strcat(filepath, "/");
|
||||||
|
strcat(filepath, entry->d_name);
|
||||||
|
|
||||||
|
if(fileIsSnapshot(filepath)) {
|
||||||
|
uint64_t timestamp = peekSnapshotFileTimestamp(filepath);
|
||||||
|
|
||||||
|
if(timestamp > highestTimestamp) {
|
||||||
|
assert(strlen(entry->d_name) < 256);
|
||||||
|
highestTimestamp = timestamp;
|
||||||
|
strcpy(newestFilename, entry->d_name);
|
||||||
|
} else if(timestamp == highestTimestamp) {
|
||||||
|
fprintf(stderr, "Duplicate snapshots (or two snapshots with same timestamp). %ld\n",
|
||||||
|
timestamp);
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
entry = readdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
closedir(dir);
|
||||||
|
|
||||||
|
if(strlen(newestFilename) > 0) {
|
||||||
|
memset(filepath, 0, 256 * 2);
|
||||||
|
strcat(filepath, snapshotDir);
|
||||||
|
strcat(filepath, "/");
|
||||||
|
strcat(filepath, newestFilename);
|
||||||
|
FILE* fp = fopen(filepath, "rb");
|
||||||
|
return fp;
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "Failed to find the latest snapshot (no snapshots found).\n");
|
||||||
|
exit(-1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FILE* openSnapshotByTimestamp(char* snapshotDir, uint64_t targetTimestamp) {
|
||||||
|
DIR* dir = opendir(snapshotDir);
|
||||||
|
char filepath[256 * 2];
|
||||||
|
|
||||||
|
struct dirent* entry = readdir(dir);
|
||||||
|
bool found = false;
|
||||||
|
while(entry != NULL) {
|
||||||
|
memset(filepath, 0, 256 * 2);
|
||||||
|
strcat(filepath, snapshotDir);
|
||||||
|
strcat(filepath, "/");
|
||||||
|
strcat(filepath, entry->d_name);
|
||||||
|
|
||||||
|
if(fileIsSnapshot(filepath)) {
|
||||||
|
if(peekSnapshotFileTimestamp(filepath) == targetTimestamp) {
|
||||||
|
found = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
entry = readdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
closedir(dir);
|
||||||
|
|
||||||
|
if(found) {
|
||||||
|
return fopen(filepath, "rb");
|
||||||
|
} else {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void loadInSnapshots(char* snapshotDir, bool newest, uint64_t fromTime) {
|
||||||
|
FILE* startPoint;
|
||||||
|
|
||||||
|
if(newest) {
|
||||||
|
startPoint = openNewestSnapshot(snapshotDir);
|
||||||
|
} else {
|
||||||
|
startPoint = openSnapshotByTimestamp(snapshotDir, fromTime);
|
||||||
|
}
|
||||||
|
|
||||||
|
latestSnapshot = makeSnapshot(startPoint);
|
||||||
|
|
||||||
|
Snapshot* iter = latestSnapshot;
|
||||||
|
while(isSnapshotComplete(iter) == false && isSnapshotInitial(iter) == false) {
|
||||||
|
FILE* fp = openSnapshotByTimestamp(snapshotDir, iter->previousTimestamp);
|
||||||
|
iter->previous = makeSnapshot(fp);
|
||||||
|
iter = iter->previous;
|
||||||
|
}
|
||||||
|
}
|
43
src/snapshot.h
Normal file
43
src/snapshot.h
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
typedef struct _Snapshot Snapshot;
|
||||||
|
|
||||||
|
typedef struct _Snapshot {
|
||||||
|
FILE* fp;
|
||||||
|
|
||||||
|
uint32_t blockSize;
|
||||||
|
|
||||||
|
uint64_t timestamp;
|
||||||
|
uint64_t previousTimestamp;
|
||||||
|
uint64_t blockCount;
|
||||||
|
|
||||||
|
Snapshot* previous;
|
||||||
|
|
||||||
|
long cursor;
|
||||||
|
size_t blockListIndex;
|
||||||
|
uint64_t blockAtCursor;
|
||||||
|
} Snapshot;
|
||||||
|
|
||||||
|
extern Snapshot* latestSnapshot;
|
||||||
|
extern uint64_t snapshotTotalBlocks;
|
||||||
|
|
||||||
|
void writeCompleteSnapshot(char* snapshotDir, char* basename, FILE* newState, bool initial);
|
||||||
|
void writePartialSnapshot(char* snapshotDir, char* basename, FILE* newState, uint64_t differenceCount);
|
||||||
|
bool fileIsSnapshot(char* filepath);
|
||||||
|
bool snapshotsExist(char* dirpath);
|
||||||
|
|
||||||
|
Snapshot* makeSnapshot(FILE* fp);
|
||||||
|
bool isSnapshotComplete(Snapshot* sp);
|
||||||
|
bool isSnapshotInitial(Snapshot* sp);
|
||||||
|
bool isSnapshotAtBlock(Snapshot* sp, uint64_t blockNum);
|
||||||
|
bool isSnapshotBlockAfter(Snapshot* sp, uint64_t blockNum);
|
||||||
|
bool isSnapshotDone(Snapshot* sp);
|
||||||
|
bool snapshotNextBlock(Snapshot* sp);
|
||||||
|
void snapshotRewindCursor(Snapshot* sp);
|
||||||
|
void snapshotRewindAllCursors();
|
||||||
|
void snapshotReadBlockAtCursor(Snapshot* sp, uint8_t* dst);
|
||||||
|
void completeSnapshotReadBlock(Snapshot* sp, uint64_t blockNum, uint8_t* dst);
|
||||||
|
void loadInSnapshots(char* snapshotDir, bool newest, uint64_t fromTime);
|
Loading…
Add table
Reference in a new issue