Andreas Romeyke
5 years ago
12 changed files with 514 additions and 7 deletions
-
81README.TIFFrisks
-
9src/CMakeLists.txt
-
1src/checkit_tiff.c
-
92src/checkit_tiff_risk.c
-
64src/headers/check.h
-
4src/headers/check_helper.h
-
4src/headers/config_parser.h
-
18src/helper/check_tiffparse.c
-
248src/risk_analyzer/lst_offset_mappings.c
-
BINtiffs_should_fail/minimal_offsets_multiple_stripoffsets.tif
-
BINtiffs_should_fail/minimal_without_IFD.tif
-
BINtiffs_should_pass/minimal_valid_multiple_stripoffsets.tif
@ -0,0 +1,81 @@ |
|||
Some thoughts about risks in TIFF file format |
|||
============================================= |
|||
|
|||
== Introduction == |
|||
|
|||
TIFF in general is a very simple fileformat. It starts with a constant header entry, which indicates that the file is a TIFF and how it is encoded (byteorder). |
|||
|
|||
The header contains an offset entry which points to the first image file directory (IFD). Each IFD has a field which counts the number of associated tags, followed by an array of these tags and an offset entry to the next IFD or to zero, which means there is no further IFD. |
|||
|
|||
Each tag in the array is 12 Bytes long. The first 4 bytes indicate the tag itself, the next 2 bytes declare the value-type, followed by 2 bytes counting the values. The last 4 bytes are either an offset or hold the values themselves. |
|||
|
|||
== What makes a TIFF robust? == |
|||
|
|||
In the TIFF specification, there are some hints which help us to repair broken TIFFs. |
|||
|
|||
The first hint is that all offset-addresses must be even. |
|||
The second importand rule is that the tags in an IFD must be sorted in an ascending order. |
|||
|
|||
At last, the TIFF spec defines different areas in the tag range. This is to guarantee that the important values are well defined. |
|||
|
|||
If we guarantee that a valid TIFF was stored, there is a good chance to detect and repair broken TIFFs using these three hints. |
|||
|
|||
== What are the caveats of TIFF? == |
|||
|
|||
As a proof of concept there is also a tool "checkit_tiff_risks" provided in this repository. Using this tool, users can analyze the layout of any baseline TIF file. |
|||
|
|||
The most risky memory ranges are the offsets. If a bitflip occurs there, the user must search the complete 4GB range. In practise, the TIF files are smaller, and so this size is the searchspace for offsets. |
|||
|
|||
The most risky offsets are the ones which are indirect offsets. This means the IFD0 offset and the StripOffset tag (code 273). |
|||
|
|||
Here an example of a possible complex StripOffset encoding: |
|||
|
|||
[ditaa] |
|||
---- |
|||
+-------------+ |
|||
| TIFF header |--\ |
|||
+-------------+ | |
|||
| |
|||
/-----------/ Offset to IFD0 |
|||
| |
|||
v |
|||
+-------------+ |
|||
| IFD0 | |
|||
|-------------| |
|||
| ... | |
|||
| StripOffset |--\ |
|||
+-------------+ | |
|||
| Offset to values |
|||
/-----------/ of tag StripOffset |
|||
| |
|||
v |
|||
+-------------+ |
|||
| StripOffs. 0|----\ |
|||
| StripOffs. 1|--\ | Offset to pixel-data |
|||
+-------------+ | | of stripe 0 |
|||
| | |
|||
/-----------/ | |
|||
| | |
|||
v | |
|||
+-------------+ | |
|||
| pixel-data 1| | |
|||
+-------------+ | |
|||
| |
|||
/-------------/ |
|||
| |
|||
v |
|||
+-------------+ |
|||
| pixel-data 1| |
|||
+-------------+ |
|||
|
|||
---- |
|||
|
|||
The problem in this example is that TIFF has no way to find out how many bytes are part of the pixel-data stream. The existing StripByteCounts tag only stores the expected pixel data length *after* decompression. |
|||
|
|||
This makes the StripOffset tag very fragile. If a bitflip changes the offset of the StripOffset tag, the whole pixel information might be lost. |
|||
|
|||
Also, if a bitflip occurs in the offset area that the StripOffset tag points to, the partial pixel data of the affected stripe is lost. |
|||
|
|||
If compression is used, the risk of losing the whole picture is even higher, because the compression methods do not use an end-symbol. Instead, the buffer sizes as stored in the StripByteCount tag are used. |
|||
Therefore, a bit-error in the Compression tag, the StripOffset tag, the StripByteCount tag or in the memory-map where StripOffset points to, could destroy the picture information. |
|||
|
|||
@ -0,0 +1,92 @@ |
|||
/* rule based checks if given TIFF is a specific baseline TIFF |
|||
* |
|||
* author: Andreas Romeyke, 2015/16 |
|||
* licensed under conditions of libtiff |
|||
* (see http://libtiff.maptools.org/misc.html) |
|||
* |
|||
*/ |
|||
|
|||
#include "check.h" |
|||
#include "check_helper.h" |
|||
#include <ctype.h> |
|||
#ifdef __unix__ |
|||
#include <unistd.h> |
|||
#include <sys/stat.h> |
|||
#include <sys/types.h> |
|||
#else |
|||
/* #include <sys\stat.h> */ |
|||
#include <sys/stat.h> |
|||
#endif |
|||
|
|||
#define FLAGGED 1 |
|||
#define UNFLAGGED 0 |
|||
|
|||
/** help function */ |
|||
void help () { |
|||
printf ("checkit_tiff_risk\n"); |
|||
printf("licensed under conditions of libtiff (see http://libtiff.maptools.org/misc.html)\n\n"); |
|||
printf ("call it with:\n"); |
|||
printf ("\tcheckit_tiff_risk [-m|-h|-s] <tifffile>\n"); |
|||
printf ("\nwhere <tifffile> is the tiff file\n"); |
|||
printf ("\t-h this help\n"); |
|||
printf ("\t-m prints a memory map\n"); |
|||
printf ("\t-s prints statistics\n"); |
|||
printf ("example:\n\tcheckit_tiff -m tiffs_should_pass/minimal_valid.tiff\n"); |
|||
printf ("\n"); |
|||
} |
|||
|
|||
|
|||
/* *********** main ************* */ |
|||
int main(int argc, char * argv[]) { |
|||
int c; |
|||
int flag_print_map = UNFLAGGED; |
|||
int flag_print_stats = UNFLAGGED; |
|||
while ((c = getopt (argc, argv, "hms")) != -1) { |
|||
switch (c) |
|||
{ |
|||
case 'h': /* help */ |
|||
help(); |
|||
exit (0); |
|||
case 'm': /* mem map */ |
|||
flag_print_map = FLAGGED; |
|||
break; |
|||
case 's': /* stats */ |
|||
flag_print_stats = FLAGGED; |
|||
break; |
|||
case '?': /* something goes wrong */ |
|||
/* |
|||
if (optopt == 'r') { |
|||
fprintf (stderr, "Option -%c requires an argument.\n", optopt); |
|||
return (-1); |
|||
} |
|||
else*/ if (isprint (optopt)) { |
|||
fprintf (stderr, "Unknown option `-%c'.\n", optopt); |
|||
return (-1); |
|||
} |
|||
else if (0 !=optopt) { |
|||
fprintf (stderr, "Unknown option character `\\x%x'.\n", optopt); |
|||
return (-1); |
|||
} |
|||
break; |
|||
default: |
|||
abort(); |
|||
} |
|||
|
|||
|
|||
} |
|||
if (argc - optind != 1) { |
|||
help(); |
|||
fprintf( stderr, "%s needs at least one arguments, example:\n\t %s tiffs_should_pass/minimal_valid.tiff\n", argv[0], argv[0]); |
|||
exit (EXIT_FAILURE); |
|||
} |
|||
const char *tiff_file=argv[optind]; |
|||
tif_files(tiff_file); |
|||
ctiff_t * ctif = initialize_ctif( tiff_file ); |
|||
mem_map_t * memmap_p = scan_mem_map(ctif); |
|||
if (FLAGGED == flag_print_map) |
|||
print_mem_map ( memmap_p ); |
|||
if (FLAGGED == flag_print_stats) |
|||
print_mem_stats( memmap_p ); |
|||
return 0; |
|||
} |
|||
|
|||
@ -0,0 +1,248 @@ |
|||
/* rule based checks if given TIFF is a specific baseline TIFF |
|||
* |
|||
* author: Andreas Romeyke, 2015/2016 |
|||
* licensed under conditions of libtiff |
|||
* (see http://libtiff.maptools.org/misc.html) |
|||
* |
|||
*/ |
|||
|
|||
#include "check.h" |
|||
#include "check_helper.h" |
|||
#include <unistd.h> |
|||
#include <assert.h> |
|||
|
|||
|
|||
void add_mem_entry(mem_map_t * memmap_p, uint32 offset, uint32 count, memtype_t type) { |
|||
mem_map_t memmap = *( memmap_p ); |
|||
assert(NULL != memmap.base_p); |
|||
assert(0 < memmap.max_entries); |
|||
assert(memmap.count < memmap.max_entries); |
|||
assert(type < mt_END_marker); |
|||
mem_map_entry_t * memmap_it_p = memmap.base_p; |
|||
memmap_it_p = memmap.base_p + memmap.count; |
|||
memmap_it_p->offset=offset; |
|||
memmap_it_p->count=count; |
|||
memmap_it_p->mem_type=type; |
|||
memmap.count++; |
|||
assert( memmap.max_entries > memmap.count); |
|||
*( memmap_p) = memmap; |
|||
} |
|||
|
|||
int compare_memmap(void const * lhs, void const * rhs) { |
|||
mem_map_entry_t left = *((mem_map_entry_t *) lhs); |
|||
mem_map_entry_t right = *((mem_map_entry_t *) rhs); |
|||
|
|||
return ((left.offset) - (right.offset)); |
|||
} |
|||
|
|||
/* TODO: add support for StripOffsets and StripByteCounts */ |
|||
mem_map_t * scan_mem_map(ctiff_t * ctif) { |
|||
assert( NULL != ctif); |
|||
static mem_map_t memmap; |
|||
memmap.count = 0; |
|||
memmap.base_p = NULL; |
|||
memmap.max_entries = 2048; |
|||
memmap.base_p = malloc (sizeof(mem_map_entry_t) * memmap.max_entries); |
|||
if (NULL == memmap.base_p) { |
|||
perror ("could not allocate mem for memmap, abort"); |
|||
} |
|||
/* size of tiff file in bytes */ |
|||
memmap.max_len = ctif->streamlen; |
|||
/* header */ |
|||
add_mem_entry( &memmap, 0, 4, mt_constant); |
|||
/* IFD0 Offset */ |
|||
add_mem_entry( &memmap, 4, 4, mt_offset_to_ifd0); |
|||
/* IFDO */ |
|||
uint32 ifd = get_ifd0_pos( ctif ); |
|||
uint32 count = get_ifd0_count( ctif); |
|||
|
|||
add_mem_entry( &memmap, ifd, 2, mt_ifd); /* count of tags in ifd */ |
|||
int ifdbase=2+ifd; |
|||
/* iterate through IFD0 entries */ |
|||
int tagidx; |
|||
ifd_entry_t stripoffset_entry; |
|||
|
|||
for (tagidx = 0; tagidx< count; tagidx++) { |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12), 8, mt_ifd); /* tagid, field type, count */ |
|||
ifd_entry_t ifd_entry = TIFFGetRawTagIFDListEntry( ctif, tagidx ); |
|||
uint32 tag = TIFFGetRawTagListEntry( ctif, tagidx); |
|||
TIFFDataType datatype = TIFFGetRawTagType( ctif, tag); |
|||
int datasize; |
|||
switch (datatype) { |
|||
case TIFF_ASCII: datasize = 1; break; |
|||
case TIFF_LONG: datasize = 4; break; |
|||
case TIFF_SHORT: datasize = 2; break; |
|||
case TIFF_BYTE: datasize = 1; break; |
|||
case TIFF_UNDEFINED: datasize = 1; break; |
|||
case TIFF_RATIONAL: datasize = 8; break; |
|||
case TIFF_SSHORT: datasize = 2; break; |
|||
case TIFF_SBYTE: datasize = 1; break; |
|||
case TIFF_SLONG: datasize = 4; break; |
|||
case TIFF_SRATIONAL: datasize = 8; break; |
|||
case TIFF_FLOAT: datasize = 4; break; |
|||
case TIFF_DOUBLE: datasize = 8; break; |
|||
} |
|||
uint32 offset = ifd_entry.data32offset; |
|||
uint16 count = ifd_entry.count; |
|||
|
|||
if (tag == TIFFTAG_STRIPOFFSETS) { |
|||
if (ifd_entry.value_or_offset == is_offset) { |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8,4,mt_ifd_offset_to_standardized_value ); |
|||
add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_ifd_offset_to_stripoffsets ); |
|||
} else if (ifd_entry.value_or_offset==is_value) { |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_standardized_value ); |
|||
} |
|||
stripoffset_entry=ifd_entry; |
|||
} else if (ifd_entry.value_or_offset==is_offset) { /* offset */ |
|||
if (tag < 32768) { /* standard tag */ |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_offset_to_standardized_value ); |
|||
add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_standardized_value ); |
|||
} else if (tag < 65000) { /* registered tag */ |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_offset_to_registered_value ); |
|||
add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_registered_value ); |
|||
} else { /* private tag */ |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_offset_to_private_value ); |
|||
add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_private_value ); |
|||
} |
|||
} else if (ifd_entry.value_or_offset==is_value) { /* embedded value */ |
|||
if (tag < 32768) { /* standard tag */ |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_standardized_value ); |
|||
} else if (tag < 65000) { /* registered tag */ |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_registered_value ); |
|||
} else { /* private tag */ |
|||
add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_private_value ); |
|||
} |
|||
|
|||
} |
|||
} |
|||
|
|||
/* check next IFD mark */ |
|||
// uint32 offset = get_ifd0_pos(ctif ); |
|||
// uint32 IFDn = get_next_ifd_pos( ctif, offset ); |
|||
// printf("IFD: offset=%i, IFD0=%i IFDn=%i ifd+count=%i\n", offset, ifd, IFDn, ifdbase+12*count); |
|||
add_mem_entry( &memmap, ifdbase+12*count, 4, mt_offset_to_ifd); |
|||
|
|||
/* handle stripoffset data */ |
|||
//printf("ifd_count=%i, stripoffset_count=%i\n", stripoffset_entry.count, stripoffset_count); |
|||
uint32 stripoffset_values[stripoffset_entry.count]; |
|||
switch (stripoffset_entry.datatype) { |
|||
case TIFF_LONG: { |
|||
/* value */ |
|||
if (stripoffset_entry.value_or_offset == is_value) { |
|||
for (int i=0; i< stripoffset_entry.count; i++) { |
|||
stripoffset_values[i] = stripoffset_entry.data32; |
|||
} |
|||
} |
|||
/* offset */ |
|||
if (stripoffset_entry.value_or_offset == is_offset) { |
|||
offset_t offset = read_offsetdata(ctif, stripoffset_entry.data32offset, stripoffset_entry.count, stripoffset_entry.datatype); |
|||
uint32 * p = offset.data32p; |
|||
for (int i=0; i< stripoffset_entry.count; i++) { |
|||
uint32 pval = *p; |
|||
if (is_byteswapped(ctif)) { |
|||
TIFFSwabLong(&pval); |
|||
} |
|||
stripoffset_values[i]=pval; |
|||
p++; |
|||
} |
|||
} |
|||
break; |
|||
} |
|||
case TIFF_SHORT: { |
|||
/* value */ |
|||
if (stripoffset_entry.value_or_offset == is_value) { |
|||
for (int i=0; i< stripoffset_entry.count; i++) { |
|||
stripoffset_values[i]= stripoffset_entry.data16[i]; |
|||
} |
|||
} |
|||
/* offset */ |
|||
if (stripoffset_entry.value_or_offset == is_offset) { |
|||
offset_t offset = read_offsetdata(ctif, stripoffset_entry.data32offset, stripoffset_entry.count, stripoffset_entry.datatype); |
|||
uint16 * p = offset.data16p; |
|||
for (int i=0; i< count; i++) { |
|||
uint16 pval = *p; |
|||
if (is_byteswapped(ctif)) { |
|||
TIFFSwabShort(&pval); |
|||
} |
|||
stripoffset_values[i]=pval; |
|||
p++; |
|||
} |
|||
} |
|||
break; |
|||
} |
|||
} |
|||
//printf("count=%i\n", stripoffset_entry.count); |
|||
for (int i=0; i< stripoffset_entry.count; i++) { |
|||
tsize_t rawstriplen = TIFFRawStripSize(ctif->tif, i); |
|||
//printf("OFFSET: p[%i]=%u len=%i\n", i,stripoffset_values[i], rawstriplen); |
|||
add_mem_entry( &memmap, stripoffset_values[i], rawstriplen, mt_stripoffset_value); |
|||
} |
|||
|
|||
/* sort entries by offset */ |
|||
qsort(memmap.base_p, memmap.count, sizeof( mem_map_entry_t), compare_memmap); |
|||
/* |
|||
printf("memmap before HOLE detection\n"); |
|||
print_mem_map( &memmap ); |
|||
printf("----------------------------\n"); |
|||
*/ |
|||
|
|||
/* add all unused areas */ |
|||
int memmap_orig_count = memmap.count; |
|||
for (int j=1; j< memmap_orig_count; j++) { |
|||
mem_map_entry_t * prev=memmap.base_p+j-1; |
|||
mem_map_entry_t * act =memmap.base_p+j; |
|||
uint32 estimated_offset = (prev->offset + prev->count); |
|||
if (estimated_offset < act->offset) { /* found a hole */ |
|||
printf("HOLE FOUND at %i\n", estimated_offset); |
|||
|
|||
printf("\tprev->offset=%i prev->count=%i estimated=%i\n", prev->offset, prev->count, estimated_offset); |
|||
printf("\tact->offset=%i act->count=%i\n", act->offset, act->count); |
|||
|
|||
add_mem_entry( &memmap, estimated_offset, (act->offset -estimated_offset), mt_unused); |
|||
} |
|||
} |
|||
/* sort entries by offset again */ |
|||
qsort(memmap.base_p, memmap.count, sizeof( mem_map_entry_t), compare_memmap); |
|||
/* add unused area at end */ |
|||
mem_map_entry_t * last = memmap.base_p + memmap.count-1; |
|||
uint32 estimated_offset = (last->offset + last->count); |
|||
if (memmap.max_len > estimated_offset) { |
|||
printf("HOLE (at end) FOUND at %i\n", estimated_offset); |
|||
add_mem_entry( &memmap, estimated_offset, (memmap.max_len -estimated_offset), mt_unused); |
|||
} |
|||
/* sort entries by offset again */ |
|||
qsort(memmap.base_p, memmap.count, sizeof( mem_map_entry_t), compare_memmap); |
|||
return &memmap; |
|||
} |
|||
|
|||
|
|||
|
|||
void print_mem_map( mem_map_t * memmap_p ) { |
|||
assert(NULL != memmap_p); |
|||
mem_map_t memmap = *( memmap_p ); |
|||
printf("size of file is %u bytes\n", memmap.max_len); |
|||
for (int j=0; j< memmap.count; j++) { |
|||
mem_map_entry_t * i=memmap.base_p+j; |
|||
printf ("entry=%03i, offset_start=%10i, count=%10i, offset_next=%10i, [%02i], type=%s\n", j,i->offset, i->count, (i->offset + i->count), i->mem_type, memtype_string[i->mem_type]); |
|||
} |
|||
} |
|||
|
|||
void print_mem_stats( mem_map_t * memmap_p) { |
|||
uint32 size = memmap_p->max_len; |
|||
uint32 stat[ mt_END_marker ]; |
|||
uint32 counted=0; |
|||
for (int i=0; i< mt_END_marker; i++) { |
|||
stat[i]=0; |
|||
} |
|||
|
|||
for (int j=0; j< memmap_p->count; j++) { |
|||
mem_map_entry_t * i=memmap_p->base_p+j; |
|||
stat[i->mem_type]+= i->count; |
|||
counted+=i->count; |
|||
} |
|||
for (int i=0; i<mt_END_marker; i++) { |
|||
float ratio = ((float) (stat[i])) / ((float) size); |
|||
printf("[%02i], type=%32s, bytes=%10i, ratio=%0.5f\n", i, memtype_string[i], stat[i], ratio); |
|||
} |
|||
printf("counted: %i bytes, size: %i bytes\n", counted, size); |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue