Merge branch 'feature_risk_analyzer'

README.TIFFrisks

@@ -0,0 +1,81 @@
+Some thoughts about risks in TIFF file format
+=============================================
+
+== Introduction ==
+
+TIFF in general is a very simple fileformat. It starts with a constant header entry, which indicates that the file is a TIFF and how it is encoded (byteorder).
+
+The header contains an offset entry which points to the first image file directory (IFD). Each IFD has a field which counts the number of associated tags, followed by an array of these tags and an offset entry to the next IFD or to zero, which means there is no further IFD.
+
+Each tag in the array is 12 Bytes long. The first 4 bytes indicate the tag itself, the next 2 bytes declare the value-type, followed by 2 bytes counting the values. The last 4 bytes are either an offset or hold the values themselves.
+
+== What makes a TIFF robust? ==
+
+In the TIFF specification, there are some hints which help us to repair broken TIFFs.
+
+The first hint is that all offset-addresses must be even.
+The second importand rule is that the tags in an IFD must be sorted in an ascending order.
+
+At last, the TIFF spec defines different areas in the tag range. This is to guarantee that the important values are well defined.
+
+If we guarantee that a valid TIFF was stored, there is a good chance to detect and repair broken TIFFs using these three hints.
+
+== What are the caveats of TIFF? ==
+
+As a proof of concept there is also a tool "checkit_tiff_risks" provided in this repository. Using this tool, users can analyze the layout of any baseline TIF file.
+
+The most risky memory ranges are the offsets. If a bitflip occurs there, the user must search the complete 4GB range. In practise, the TIF files are smaller, and so this size is the searchspace for offsets.
+
+The most risky offsets are the ones which are indirect offsets. This means the IFD0 offset and the StripOffset tag (code 273).
+
+Here an example of a possible complex StripOffset encoding:
+
+[ditaa]
+----
++-------------+
+| TIFF header |--\
++-------------+  |
+                 |
+     /-----------/ Offset to IFD0
+     |
+     v
++-------------+
+| IFD0        |
+|-------------|
+| ...         |
+| StripOffset |--\
++-------------+  |
+                 | Offset to values
+     /-----------/ of tag StripOffset
+     |
+     v
++-------------+
+| StripOffs. 0|----\
+| StripOffs. 1|--\ | Offset to pixel-data
++-------------+  | | of stripe 0
+                 | |
+     /-----------/ |
+     |             |
+     v             |
++-------------+    |
+| pixel-data 1|    |
++-------------+    |
+                   |
+     /-------------/
+     |
+     v
++-------------+
+| pixel-data 1|
++-------------+
+
+----
+
+The problem in this example is that TIFF has no way to find out how many bytes are part of the pixel-data stream. The existing StripByteCounts tag only stores the expected pixel data length *after* decompression.
+
+This makes the StripOffset tag very fragile. If a bitflip changes the offset of the StripOffset tag, the whole pixel information might be lost.
+
+Also, if a bitflip occurs in the offset area that the StripOffset tag points to, the partial pixel data of the affected stripe is lost.
+
+If compression is used, the risk of losing the whole picture is even higher, because the compression methods do not use an end-symbol. Instead, the buffer sizes as stored in the StripByteCount tag are used.
+Therefore, a bit-error in the Compression tag, the StripOffset tag, the StripByteCount tag or in the memory-map where StripOffset points to, could destroy the picture information.
+

src/CMakeLists.txt

@@ -10,7 +10,14 @@ file( GLOB checkit_tiff_SOURCES
   "${PROJECT_SOURCE_DIR}/helper/*.c"
   "${PROJECT_SOURCE_DIR}/parser/*parser.c"
   "${PROJECT_SOURCE_DIR}/validate_icc/*.c"
+  "${PROJECT_SOURCE_DIR}/risk_analyzer/*.c"
   )
+
+file( GLOB checkit_tiff_risk_SOURCES
+  "${PROJECT_SOURCE_DIR}/helper/*.c"
+  "${PROJECT_SOURCE_DIR}/risk_analyzer/*.c"
+)
+
 #
 # Make a version file containing the current version from git.
 #
@@ -33,6 +40,7 @@ endif(WIN32)
 
 add_executable(checkit_tiff checkit_tiff.c ${checkit_tiff_SOURCES})
 add_executable(checkit_check_config checkit_check_config.c ${checkit_tiff_SOURCES})
+add_executable(checkit_tiff_risk checkit_tiff_risk.c ${checkit_tiff_risk_SOURCES})
 set(CMAKE_EXTRA_INCLUDE_FILES tiff.h)
 
 
@@ -106,6 +114,7 @@ endif(PCRE_FOUND)
 
 TARGET_LINK_LIBRARIES(checkit_tiff ${TIFF_LIBRARIES}  ${PCRE_LIBRARIES})
 TARGET_LINK_LIBRARIES(checkit_check_config ${TIFF_LIBRARIES}  ${PCRE_LIBRARIES})
+TARGET_LINK_LIBRARIES(checkit_tiff_risk ${TIFF_LIBRARIES})
 
 install( TARGETS checkit_tiff
   RUNTIME DESTINATION bin

src/checkit_tiff.c

@@ -57,6 +57,7 @@ void simplified_result_push(ret_t res, function_t func) {
         result_push( full );
 }
 
+
 int check_specific_tiff_file( const char * tiff_file, int use_memmapped) {
   // printf("tiff file=%s\n", tiff_file);
   ret_t res = _empty_result();

src/checkit_tiff_risk.c

@@ -0,0 +1,92 @@
+/* rule based checks if given TIFF is a specific baseline TIFF
+ * 
+ * author: Andreas Romeyke, 2015/16
+ * licensed under conditions of libtiff 
+ * (see http://libtiff.maptools.org/misc.html)
+ *
+ */
+
+#include "check.h"
+#include "check_helper.h"
+#include <ctype.h>
+#ifdef __unix__
+      #include <unistd.h>
+      #include <sys/stat.h>
+      #include <sys/types.h>
+#else
+      /* #include <sys\stat.h> */
+      #include <sys/stat.h>
+#endif
+
+#define FLAGGED 1
+#define UNFLAGGED 0
+
+/** help function */
+void help () {
+  printf ("checkit_tiff_risk\n");
+  printf("licensed under conditions of libtiff (see http://libtiff.maptools.org/misc.html)\n\n");
+  printf ("call it with:\n");
+  printf ("\tcheckit_tiff_risk [-m|-h|-s] <tifffile>\n");
+  printf ("\nwhere <tifffile> is the tiff file\n");
+  printf ("\t-h this help\n");
+  printf ("\t-m prints a memory map\n");
+  printf ("\t-s prints statistics\n");
+  printf ("example:\n\tcheckit_tiff -m tiffs_should_pass/minimal_valid.tiff\n");
+  printf ("\n");
+}
+
+
+/* *********** main ************* */
+int main(int argc, char * argv[]) {
+  int c;
+  int flag_print_map = UNFLAGGED;
+  int flag_print_stats = UNFLAGGED;
+  while ((c = getopt (argc, argv, "hms")) != -1) {
+    switch (c)
+    {
+      case 'h': /* help */
+        help();
+        exit (0);
+      case 'm': /*  mem map */
+        flag_print_map = FLAGGED;
+        break;
+      case 's': /*  stats */
+        flag_print_stats = FLAGGED;
+        break;
+      case '?': /* something goes wrong */
+        /*
+           if (optopt == 'r') {
+           fprintf (stderr, "Option -%c requires an argument.\n", optopt);
+           return (-1);
+           }
+           else*/ if (isprint (optopt)) {
+             fprintf (stderr, "Unknown option `-%c'.\n", optopt);
+             return (-1);
+           }
+           else if (0 !=optopt) {
+             fprintf (stderr, "Unknown option character `\\x%x'.\n", optopt);
+             return (-1);
+           }
+           break;
+      default:
+           abort();
+    }
+
+
+  }
+  if (argc - optind != 1) {
+    help();
+    fprintf( stderr, "%s needs at least one arguments, example:\n\t %s tiffs_should_pass/minimal_valid.tiff\n", argv[0], argv[0]);
+    exit (EXIT_FAILURE);
+  }
+  const char *tiff_file=argv[optind];
+  tif_files(tiff_file);
+  ctiff_t * ctif = initialize_ctif( tiff_file );
+  mem_map_t * memmap_p = scan_mem_map(ctif);
+  if (FLAGGED == flag_print_map) 
+    print_mem_map ( memmap_p );
+  if (FLAGGED == flag_print_stats)
+    print_mem_stats( memmap_p );
+  return 0;
+}
+

src/headers/check.h

@@ -70,6 +70,59 @@ typedef struct offset_s {
 
 typedef uint16 tag_t;
 
+typedef enum {
+	mt_unused, /* memory areas, which are not referenced within TIFF file */
+	mt_constant, /* constant values, which are fix part of TIFF file */
+	mt_ifd, /* memory areas, which are parts of the IFD (but no value!) */
+	mt_offset_to_ifd0,
+	mt_offset_to_ifd, /* offset to nex ifd */
+	mt_ifd_embedded_standardized_value, /* memory areas, with standardized values embedded in ifd */
+	mt_ifd_embedded_registered_value, /* memory areas, with registered values embedded in ifd */
+	mt_ifd_embedded_private_value, /* memory areas, with private values embedded in ifd */
+	mt_ifd_offset_to_standardized_value, /* memory areas, which points to standardized values */
+	mt_ifd_offset_to_registered_value, /* memory areas, which points to registered values */
+	mt_ifd_offset_to_private_value, /* memory areas, which points to private values */
+        mt_ifd_offset_to_stripoffsets, /* offsets which points to stripoffsets */
+        mt_stripoffset_value, /* offsets which points to stripoffset values, hint: if compression is used stripbytecounts holds only decompressed values! */
+	mt_standardized_value, /* memory areas, which contains standardized values */
+	mt_registered_value, /* memory areas, which contains registered values */
+	mt_private_value, /* memory areas, which contains private values */
+	mt_END_marker
+} memtype_t;
+
+static const char* memtype_string[] = {
+	"unused/unknown", /* memory areas", which are not referenced within TIFF file */
+	"constant", /* constant values", which are fix part of TIFF file */
+	"ifd", /* memory areas", which are parts of the IFD (but no value!) */
+	"offset_to_ifd0", /* offset to nex ifd */
+	"offset_to_ifd", /* offset to nex ifd */
+	"ifd_embedded_standardized_value", /* memory areas", with standardized values embedded in ifd */
+	"ifd_embedded_registered_value", /* memory areas", with registered values embedded in ifd */
+	"ifd_embedded_private_value", /* memory areas", with private values embedded in ifd */
+	"ifd_offset_to_standardized_value", /* memory areas", which points to standardized values */
+	"ifd_offset_to_registered_value", /* memory areas", which points to registered values */
+	"ifd_offset_to_private_value", /* memory areas", which points to private values */
+        "ifd_offset_to_stripoffsets", /* offsets which points to stripoffsets */
+        "stripoffset_value", /* stripoffset values, hint: if compression is used stripbytecounts holds only decompressed values! */
+	"standardized_value", /* memory areas", which contains standardized values */
+	"registered_value", /* memory areas", which contains registered values */
+	"private_value", /* memory areas", which contains private values */
+};
+
+typedef struct mem_map_entry_s {
+	uint32 offset; /* adress within the tiff */
+	uint32 count; /* count of bytes beginning with offset */
+	memtype_t mem_type; /* type of memory */
+} mem_map_entry_t;
+
+typedef struct mem_map_s {
+	int count;
+	mem_map_entry_t * base_p;
+	uint32 max_len; /* TIFF length */
+	int max_entries;
+} mem_map_t;
+
+
 #define MAXSTRLEN 1024
 #define EXPECTSTRLEN 160
 #define VALUESTRLEN 160
@@ -97,19 +150,24 @@ ret_t check_tag_has_some_of_these_values( ctiff_t * ctif, tag_t tag, int count,
 ret_t check_tag_has_valuelist( ctiff_t * ctif, tag_t tag, int count, unsigned int * values);
 ret_t check_tag_has_value_in_range(ctiff_t * ctif, tag_t tag, unsigned int a, unsigned int b);
 ret_t check_tag_has_value(ctiff_t * ctif, tag_t tag, unsigned int value);
-ret_t check_tag_has_value_quiet(ctiff_t * ctif, tag_t tag, unsigned int value);
+ret_t check_tag_has_value_quiet(ctiff_t * ctif, tag_t tag, unsigned int expected_value);
 ret_t check_tag(ctiff_t * ctif, tag_t tag);
 ret_t check_tag_quiet(ctiff_t * ctif, tag_t tag);
 ret_t check_notag(ctiff_t * ctif, tag_t tag);
-ret_t check_tag_has_valid_type(ctiff_t * ctiff, tag_t tag);
+ret_t check_tag_has_valid_type(ctiff_t * ctif, tag_t tag);
 ret_t check_datetime(ctiff_t * ctif);
 ret_t check_icc(ctiff_t * ctif);
 ret_t check_has_only_one_ifd(ctiff_t * ctif);
 ret_t check_tagorder(ctiff_t * ctif);
 ret_t check_tag_has_valid_asciivalue(ctiff_t * ctif, tag_t tag); 
-ret_t check_tag_has_value_matching_regex(ctiff_t * ctif, tag_t tag, const char* value);
+ret_t check_tag_has_value_matching_regex(ctiff_t * ctif, tag_t tag, const char* regex_string);
 ret_t check_all_offsets_are_word_aligned(ctiff_t * ctif);
 ret_t check_all_offsets_are_used_once_only(ctiff_t * ctif);
 ret_t check_all_IFDs_are_word_aligned(ctiff_t * ctif);
+
+
+mem_map_t * scan_mem_map(ctiff_t * ctif) ;
+void print_mem_map( mem_map_t * memmap_p);
+void print_mem_stats( mem_map_t * memmap_p);
 #endif
 /* _FIXIT_TIFF_CHECK */

src/headers/check_helper.h

@@ -21,7 +21,7 @@ ret_t check_tag_has_u32value(ctiff_t * ctif, tag_t tag, uint32 value);
 const char * TIFFTagName (tag_t tag);
 uint32 TIFFGetRawTagTypeListEntry( ctiff_t * ctif, int tagidx );
 tag_t TIFFGetRawTagListEntry( ctiff_t * ctif, int tagidx ) ;
-int TIFFGetRawTagListCount (ctiff_t * ctif, uint32 ifpos) ;
+int TIFFGetRawTagListCount (ctiff_t * ctif, uint32 ifdpos) ;
 TIFFDataType TIFFGetRawTagType(ctiff_t * ctif, tag_t tag);
 ifd_entry_t TIFFGetRawIFDEntry( ctiff_t * ctif, tag_t tag);
 ifd_entry_t TIFFGetRawTagIFDListEntry( ctiff_t * ctif, int tagidx );
@@ -33,7 +33,7 @@ const char * renderer ( const ret_t ret );
 void set_renderer_to_ansi();
 void set_renderer_to_xml();
 char * secstrcat (char * dest, const char * src, int maxsize);
-ctiff_t * initialize_ctif( const char * filename );
+ctiff_t * initialize_ctif( const char * tiff_file );
 void free_ctif( ctiff_t * ctif);
 uint32 get_ifd0_pos( ctiff_t * ctif );
 uint32 get_next_ifd_pos( ctiff_t * ctif, uint32 actual_pos );

src/headers/config_parser.h

@@ -107,8 +107,8 @@ void print_plan ();
 int print_plan_results ();
 void clean_plan ();
 void parse_plan ();
-void parse_plan_via_stream (FILE * stream);
-void parse_plan_via_file (const char * filename);
+void parse_plan_via_stream (FILE * file);
+void parse_plan_via_file (const char * cfg_file);
 void add_default_rules_to_plan();
 void set_include( const char * );
 

src/helper/check_tiffparse.c

@@ -1066,6 +1066,24 @@ ctiff_t * initialize_ctif(const char * tiff_file) {
 	}
 	ctif->tif = tif;
 	ctif->filename = strdup(tiff_file);
+	/* streamlen */
+	thandle_t client = TIFFClientdata(ctif->tif);
+	TIFFReadWriteProc readproc = TIFFGetReadProc(ctif->tif);
+	TIFFSeekProc seekproc = TIFFGetSeekProc(ctif->tif);
+	TIFFSizeProc sizeproc = TIFFGetSizeProc(ctif->tif);
+	if (! seekproc) {
+		perror ("could not get TIFFGetSeekProc");
+	}
+	if (! readproc) {
+		perror ("could not get TIFFGetReadProc");
+	}
+	if (! sizeproc) {
+		perror ("could not get TIFFGetSizeProc");
+	}
+
+
+	ctif->streamlen = sizeproc(client);
+
 	ctif->ifd0p=NULL;
 	ctif->ifd0pos= 0;
 	ctif->ifd0c= 0;

src/risk_analyzer/lst_offset_mappings.c

@@ -0,0 +1,248 @@
+/* rule based checks if given TIFF is a specific baseline TIFF
+ * 
+ * author: Andreas Romeyke, 2015/2016
+ * licensed under conditions of libtiff 
+ * (see http://libtiff.maptools.org/misc.html)
+ *
+ */
+
+#include "check.h"
+#include "check_helper.h"
+#include <unistd.h>
+#include <assert.h>
+
+
+void add_mem_entry(mem_map_t * memmap_p, uint32 offset, uint32 count, memtype_t type) {
+  mem_map_t memmap = *( memmap_p );
+  assert(NULL != memmap.base_p);
+  assert(0 < memmap.max_entries);
+  assert(memmap.count < memmap.max_entries);
+  assert(type < mt_END_marker);
+  mem_map_entry_t * memmap_it_p = memmap.base_p;
+  memmap_it_p = memmap.base_p + memmap.count;
+  memmap_it_p->offset=offset;
+  memmap_it_p->count=count;
+  memmap_it_p->mem_type=type;
+  memmap.count++;
+  assert( memmap.max_entries > memmap.count);
+  *( memmap_p) = memmap;
+}
+
+int compare_memmap(void const * lhs, void const * rhs) {
+	mem_map_entry_t left = *((mem_map_entry_t *) lhs);
+	mem_map_entry_t right = *((mem_map_entry_t *) rhs);
+
+	return ((left.offset) - (right.offset));
+}
+
+/* TODO: add support for StripOffsets and StripByteCounts */
+mem_map_t * scan_mem_map(ctiff_t * ctif) {
+  assert( NULL != ctif);
+  static mem_map_t memmap;
+  memmap.count = 0;
+  memmap.base_p = NULL;
+  memmap.max_entries = 2048;
+  memmap.base_p = malloc (sizeof(mem_map_entry_t) * memmap.max_entries);
+  if (NULL == memmap.base_p) {
+	  perror ("could not allocate mem for memmap, abort");
+  }
+  /* size of tiff file in bytes */
+  memmap.max_len = ctif->streamlen;
+  /* header */
+  add_mem_entry( &memmap, 0, 4, mt_constant);
+  /* IFD0 Offset */
+  add_mem_entry( &memmap, 4, 4, mt_offset_to_ifd0);
+  /* IFDO */
+  uint32 ifd = get_ifd0_pos( ctif );
+  uint32 count = get_ifd0_count( ctif);
+
+  add_mem_entry( &memmap, ifd, 2, mt_ifd); /* count of tags in ifd */
+  int ifdbase=2+ifd;
+  /* iterate through IFD0 entries */
+  int tagidx;
+  ifd_entry_t stripoffset_entry; 
+
+  for (tagidx = 0; tagidx< count; tagidx++) {
+	  add_mem_entry( &memmap, ifdbase+(tagidx*12), 8, mt_ifd); /* tagid, field type, count */
+	  ifd_entry_t ifd_entry = TIFFGetRawTagIFDListEntry( ctif, tagidx );
+	  uint32 tag = TIFFGetRawTagListEntry( ctif, tagidx);
+	  TIFFDataType datatype = TIFFGetRawTagType( ctif, tag);
+	  int datasize;
+	  switch (datatype) {
+		  case TIFF_ASCII: datasize = 1; break;
+		  case TIFF_LONG: datasize =  4; break;
+		  case TIFF_SHORT: datasize =  2; break;
+		  case TIFF_BYTE: datasize =  1; break;
+		  case TIFF_UNDEFINED: datasize =  1; break;
+		  case TIFF_RATIONAL: datasize =  8; break;
+		  case TIFF_SSHORT: datasize =  2; break;
+		  case TIFF_SBYTE: datasize =  1; break;
+		  case TIFF_SLONG: datasize =  4; break;
+		  case TIFF_SRATIONAL: datasize =  8; break;
+		  case TIFF_FLOAT: datasize =  4; break;
+		  case TIFF_DOUBLE: datasize =  8; break;
+	  }
+          uint32 offset = ifd_entry.data32offset;
+          uint16 count = ifd_entry.count;
+
+          if (tag == TIFFTAG_STRIPOFFSETS) {
+                  if (ifd_entry.value_or_offset == is_offset) {
+                  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8,4,mt_ifd_offset_to_standardized_value ); 
+                  add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_ifd_offset_to_stripoffsets );
+                  } else if (ifd_entry.value_or_offset==is_value) {
+                        add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_standardized_value );
+                  }
+                  stripoffset_entry=ifd_entry;
+          } else if (ifd_entry.value_or_offset==is_offset) { /* offset */
+		  if (tag < 32768) { /* standard tag */
+			  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_offset_to_standardized_value ); 
+			  add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_standardized_value );
+		  } else if (tag < 65000) { /* registered tag */
+			  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_offset_to_registered_value ); 
+			  add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_registered_value );
+		  } else { /* private tag */
+			  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_offset_to_private_value ); 
+			  add_mem_entry( &memmap, offset, ((uint32) count)*datasize, mt_private_value );
+		  }
+	  } else if (ifd_entry.value_or_offset==is_value) { /* embedded value */
+		  if (tag < 32768) { /* standard tag */
+			  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_standardized_value ); 
+		  } else if (tag < 65000) { /* registered tag */
+			  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_registered_value ); 
+		  } else { /* private tag */
+			  add_mem_entry( &memmap, ifdbase+(tagidx*12)+8, 4,mt_ifd_embedded_private_value ); 
+		  }
+
+	  }
+  }
+ 
+  /* check next IFD mark */
+  // uint32 offset = get_ifd0_pos(ctif );
+  // uint32 IFDn = get_next_ifd_pos( ctif, offset );
+  // printf("IFD: offset=%i, IFD0=%i IFDn=%i ifd+count=%i\n", offset, ifd, IFDn, ifdbase+12*count);
+  add_mem_entry( &memmap, ifdbase+12*count, 4, mt_offset_to_ifd);
+
+  /* handle stripoffset data */
+  //printf("ifd_count=%i, stripoffset_count=%i\n", stripoffset_entry.count, stripoffset_count);
+  uint32 stripoffset_values[stripoffset_entry.count];
+  switch (stripoffset_entry.datatype) {
+    case TIFF_LONG: {
+                      /*  value */
+                      if (stripoffset_entry.value_or_offset == is_value) {
+                        for (int i=0; i< stripoffset_entry.count; i++) {
+                          stripoffset_values[i] = stripoffset_entry.data32;
+                        }
+                      }
+                      /*  offset */
+                      if (stripoffset_entry.value_or_offset == is_offset) {
+                        offset_t offset = read_offsetdata(ctif, stripoffset_entry.data32offset, stripoffset_entry.count, stripoffset_entry.datatype);
+                        uint32 * p = offset.data32p;
+                        for (int i=0; i< stripoffset_entry.count; i++) {
+                          uint32 pval = *p;
+                          if (is_byteswapped(ctif)) {
+                            TIFFSwabLong(&pval);
+                          }
+                          stripoffset_values[i]=pval;
+                          p++;
+                        }
+                      }
+                      break;
+                    }
+    case TIFF_SHORT: {
+                       /*  value */
+                       if (stripoffset_entry.value_or_offset == is_value) {
+                         for (int i=0; i< stripoffset_entry.count; i++) {
+                           stripoffset_values[i]= stripoffset_entry.data16[i];
+                         }
+                       }
+                       /*  offset */
+                       if (stripoffset_entry.value_or_offset == is_offset) {
+                         offset_t offset = read_offsetdata(ctif, stripoffset_entry.data32offset, stripoffset_entry.count, stripoffset_entry.datatype);
+                         uint16 * p = offset.data16p;
+                         for (int i=0; i< count; i++) {
+                           uint16 pval = *p;
+                           if (is_byteswapped(ctif)) {
+                             TIFFSwabShort(&pval);
+                           }
+                           stripoffset_values[i]=pval;
+                           p++;
+                         }
+                       }
+                       break;
+                     }
+  }
+  //printf("count=%i\n", stripoffset_entry.count);
+  for (int i=0; i< stripoffset_entry.count; i++) {
+    tsize_t rawstriplen = TIFFRawStripSize(ctif->tif, i);
+    //printf("OFFSET: p[%i]=%u len=%i\n", i,stripoffset_values[i], rawstriplen);
+    add_mem_entry( &memmap, stripoffset_values[i], rawstriplen, mt_stripoffset_value);
+  }
+
+  /* sort entries by offset */
+  qsort(memmap.base_p, memmap.count, sizeof( mem_map_entry_t), compare_memmap);
+  /*
+  printf("memmap before HOLE detection\n");
+  print_mem_map( &memmap );
+  printf("----------------------------\n");
+  */
+
+  /* add all unused areas */
+  int memmap_orig_count = memmap.count;
+  for (int j=1; j< memmap_orig_count; j++) {
+    mem_map_entry_t * prev=memmap.base_p+j-1;
+    mem_map_entry_t * act =memmap.base_p+j;
+    uint32 estimated_offset = (prev->offset + prev->count);
+    if (estimated_offset < act->offset) { /*  found a hole */
+      printf("HOLE FOUND at %i\n", estimated_offset);
+      
+      printf("\tprev->offset=%i prev->count=%i estimated=%i\n", prev->offset, prev->count, estimated_offset);
+      printf("\tact->offset=%i act->count=%i\n", act->offset, act->count);
+      
+      add_mem_entry( &memmap, estimated_offset, (act->offset -estimated_offset), mt_unused);
+    }
+  }
+  /* sort entries by offset again */
+  qsort(memmap.base_p, memmap.count, sizeof( mem_map_entry_t), compare_memmap);
+  /*  add unused area at end */
+  mem_map_entry_t * last = memmap.base_p + memmap.count-1;
+  uint32 estimated_offset = (last->offset + last->count);
+  if (memmap.max_len > estimated_offset) {
+      printf("HOLE (at end) FOUND at %i\n", estimated_offset);
+      add_mem_entry( &memmap, estimated_offset, (memmap.max_len -estimated_offset), mt_unused);
+  }
+  /* sort entries by offset again */
+  qsort(memmap.base_p, memmap.count, sizeof( mem_map_entry_t), compare_memmap);
+  return &memmap;
+}
+
+
+
+void print_mem_map( mem_map_t * memmap_p ) {
+	assert(NULL != memmap_p);
+	mem_map_t memmap = *( memmap_p );
+	printf("size of file is %u bytes\n", memmap.max_len);
+	for (int j=0; j< memmap.count; j++) {
+		mem_map_entry_t * i=memmap.base_p+j;
+		printf ("entry=%03i, offset_start=%10i, count=%10i, offset_next=%10i, [%02i], type=%s\n", j,i->offset, i->count, (i->offset + i->count), i->mem_type, memtype_string[i->mem_type]);
+	}
+}
+
+void print_mem_stats( mem_map_t * memmap_p) {
+	uint32 size = memmap_p->max_len;
+	uint32 stat[ mt_END_marker ];
+	uint32 counted=0;
+	for (int i=0; i< mt_END_marker; i++) {
+		stat[i]=0;
+	}
+
+	for (int j=0; j< memmap_p->count; j++) {
+		mem_map_entry_t * i=memmap_p->base_p+j;
+		stat[i->mem_type]+= i->count;
+		counted+=i->count;
+	}
+	for (int i=0; i<mt_END_marker; i++) {
+		float ratio = ((float) (stat[i])) / ((float) size);
+		printf("[%02i], type=%32s, bytes=%10i, ratio=%0.5f\n", i, memtype_string[i], stat[i], ratio);
+	}
+	printf("counted: %i bytes, size: %i bytes\n", counted, size);
+}