httpd improvements: added persistent connections, improved asynchronous read mode, compile-time deflate compression, new CGI mode, SSI handler can be called with tag string (instead of tag index)

2026-05-25 01:27:05 +08:00 · 2016-04-26 13:00:27 +02:00
parent 7d77a52ea5
commit 025d5591eb
8 changed files with 923 additions and 288 deletions
--- a/src/apps/httpd/makefsdata/makefsdata.c
+++ b/src/apps/httpd/makefsdata/makefsdata.c
@@ -21,6 +21,49 @@
 #endif
 #include <dos.h>
 #include <string.h>
+#include <time.h>
+#include <sys/stat.h>
+
+/** Makefsdata can generate *all* files deflate-compressed (where file size shrinks).
+ * Since nearly all browsers support this, this is a good way to reduce ROM size.
+ * To compress the files, "miniz.c" must be downloaded seperately.
+ */
+#ifndef MAKEFS_SUPPORT_DEFLATE
+#define MAKEFS_SUPPORT_DEFLATE 0
+#endif
+
+#define COPY_BUFSIZE (1024*1024) /* 1 MByte */
+
+#if MAKEFS_SUPPORT_DEFLATE
+#include "../miniz.c"
+
+typedef unsigned char uint8;
+typedef unsigned short uint16;
+typedef unsigned int uint;
+
+#define my_max(a,b) (((a) > (b)) ? (a) : (b))
+#define my_min(a,b) (((a) < (b)) ? (a) : (b))
+
+/* COMP_OUT_BUF_SIZE is the size of the output buffer used during compression.
+   COMP_OUT_BUF_SIZE must be >= 1 and <= OUT_BUF_SIZE */
+#define COMP_OUT_BUF_SIZE COPY_BUFSIZE
+
+/* OUT_BUF_SIZE is the size of the output buffer used during decompression.
+   OUT_BUF_SIZE must be a power of 2 >= TINFL_LZ_DICT_SIZE (because the low-level decompressor not only writes, but reads from the output buffer as it decompresses) */
+#define OUT_BUF_SIZE COPY_BUFSIZE
+static uint8 s_outbuf[OUT_BUF_SIZE];
+static uint8 s_checkbuf[OUT_BUF_SIZE];
+
+/* tdefl_compressor contains all the state needed by the low-level compressor so it's a pretty big struct (~300k).
+   This example makes it a global vs. putting it on the stack, of course in real-world usage you'll probably malloc() or new it. */
+tdefl_compressor g_deflator;
+tinfl_decompressor g_inflator;
+
+int deflate_level = 10; /* default compression level, can be changed via command line */
+#define USAGE_ARG_DEFLATE " [-defl]"
+#else /* MAKEFS_SUPPORT_DEFLATE */
+#define USAGE_ARG_DEFLATE ""
+#endif /* MAKEFS_SUPPORT_DEFLATE */

 /* Compatibility defines Win32 vs. DOS */
 #ifdef WIN32
@@ -66,6 +109,7 @@
 #define LWIP_HTTPD_DYNAMIC_HEADERS 1
 #define LWIP_HTTPD_SSI             1
 #include "../httpd_structs.h"
+#include "lwip/apps/fs.h"

 #include "../core/inet_chksum.c"
 #include "../core/def.c"
@@ -85,8 +129,6 @@ static int payload_alingment_dummy_counter = 0;

 #define MAX_PATH_LEN 256

-#define COPY_BUFSIZE 10240
-
 struct file_entry
 {
   struct file_entry* next;
@@ -95,14 +137,13 @@ struct file_entry

 int process_sub(FILE *data_file, FILE *struct_file);
 int process_file(FILE *data_file, FILE *struct_file, const char *filename);
-int file_write_http_header(FILE *data_file, const char *filename, int file_size,
-                           u16_t *http_hdr_len, u16_t *http_hdr_chksum);
+int file_write_http_header(FILE *data_file, const char *filename, int file_size, u16_t *http_hdr_len,
+                           u16_t *http_hdr_chksum, u8_t provide_content_len, int is_compressed);
 int file_put_ascii(FILE *file, const char *ascii_string, int len, int *i);
 int s_put_ascii(char *buf, const char *ascii_string, int len, int *i);
 void concat_files(const char *file1, const char *file2, const char *targetfile);
 int check_path(char* path, size_t size);

-static unsigned char file_buffer_raw[COPY_BUFSIZE];
 /* 5 bytes per char + 3 bytes per line */
 static char file_buffer_c[COPY_BUFSIZE * 5 + ((COPY_BUFSIZE / HEX_BYTES_PER_LINE) * 3)];

@@ -115,13 +156,19 @@ unsigned char includeHttpHeader = 1;
 unsigned char useHttp11 = 0;
 unsigned char supportSsi = 1;
 unsigned char precalcChksum = 0;
+unsigned char includeLastModified = 0;
+#if MAKEFS_SUPPORT_DEFLATE
+unsigned char deflateNonSsiFiles = 0;
+size_t deflatedBytesReduced = 0;
+size_t overallDataBytes = 0;
+#endif

 struct file_entry* first_file = NULL;
 struct file_entry* last_file = NULL;

 static void print_usage(void)
 {
-  printf(" Usage: htmlgen [targetdir] [-s] [-i] [-f:<filename>]" NEWLINE NEWLINE);
+  printf(" Usage: htmlgen [targetdir] [-s] [-i] [-f:<filename>] [-m]" USAGE_ARG_DEFLATE NEWLINE NEWLINE);
  printf("   targetdir: relative or absolute path to files to convert" NEWLINE);
  printf("   switch -s: toggle processing of subdirectories (default is on)" NEWLINE);
  printf("   switch -e: exclude HTTP header from file (header is created at runtime, default is off)" NEWLINE);
@@ -129,6 +176,11 @@ static void print_usage(void)
  printf("   switch -nossi: no support for SSI (cannot calculate Content-Length for SSI)" NEWLINE);
  printf("   switch -c: precalculate checksums for all pages (default is off)" NEWLINE);
  printf("   switch -f: target filename (default is \"fsdata.c\")" NEWLINE);
+  printf("   switch -m: include \"Last-Modified\" header based on file time" NEWLINE);
+#if MAKEFS_SUPPORT_DEFLATE
+  printf("   switch -defl: deflate-compress all non-SSI files" NEWLINE);
+  printf("                 ATTENTION: browser has to support \"Content-Encoding: deflate\"!" NEWLINE);
+#endif
  printf("   if targetdir not specified, htmlgen will attempt to" NEWLINE);
  printf("   process files in subdirectory 'fs'" NEWLINE);
 }
@@ -171,6 +223,27 @@ int main(int argc, char *argv[])
        strncpy(targetfile, &argv[i][3], sizeof(targetfile) - 1);
        targetfile[sizeof(targetfile) - 1] = 0;
        printf("Writing to file \"%s\"\n", targetfile);
+      } else if (strstr(argv[i], "-m")) {
+        includeLastModified = 1;
+      } else if (strstr(argv[i], "-defl")) {
+#if MAKEFS_SUPPORT_DEFLATE
+        char* colon = strstr(argv[i], ":");
+        if (colon) {
+          if (colon[1] != 0) {
+            int defl_level = atoi(&colon[1]);
+            if ((defl_level >= 0) && (defl_level <= 10)) {
+              deflate_level = defl_level;
+            } else {
+              printf("ERROR: deflate level must be [0..10]" NEWLINE);
+              exit(0);
+            }
+          }
+        }
+        deflateNonSsiFiles = 1;
+        printf("Deflating all non-SSI files with level %d (but only if size is reduced)" NEWLINE, deflate_level);
+#else
+        printf("WARNING: Deflate support is disabled\n");
+#endif
      } else if ((strstr(argv[i], "-?")) || (strstr(argv[i], "-h"))) {
        print_usage();
        exit(0);
@@ -230,6 +303,20 @@ int main(int argc, char *argv[])
  fprintf(data_file, "#include \"fsdata.h\"" NEWLINE NEWLINE NEWLINE);

  fprintf(data_file, "#define file_NULL (struct fsdata_file *) NULL" NEWLINE NEWLINE NEWLINE);
+  /* define FS_FILE_FLAGS_HEADER_INCLUDED to 1 if not defined (compatibility with older httpd/fs) */
+  fprintf(data_file, "#ifndef FS_FILE_FLAGS_HEADER_INCLUDED" NEWLINE "#define FS_FILE_FLAGS_HEADER_INCLUDED 1" NEWLINE "#endif" NEWLINE);
+  /* define FS_FILE_FLAGS_HEADER_PERSISTENT to 0 if not defined (compatibility with older httpd/fs: wasn't supported back then) */
+  fprintf(data_file, "#ifndef FS_FILE_FLAGS_HEADER_PERSISTENT" NEWLINE "#define FS_FILE_FLAGS_HEADER_PERSISTENT 0" NEWLINE "#endif" NEWLINE);
+
+  /* define alignment defines */
+#if ALIGN_PAYLOAD
+  fprintf(data_file, "/* FSDATA_FILE_ALIGNMENT: 0=off, 1=by variable, 2=by include */" NEWLINE "#ifndef FSDATA_FILE_ALIGNMENT" NEWLINE "#define FSDATA_FILE_ALIGNMENT 0" NEWLINE "#endif" NEWLINE);
+#endif
+  fprintf(data_file, "#ifndef FSDATA_ALIGN_PRE"  NEWLINE "#define FSDATA_ALIGN_PRE"  NEWLINE "#endif" NEWLINE);
+  fprintf(data_file, "#ifndef FSDATA_ALIGN_POST" NEWLINE "#define FSDATA_ALIGN_POST" NEWLINE "#endif" NEWLINE);
+#if ALIGN_PAYLOAD
+  fprintf(data_file, "#if FSDATA_FILE_ALIGNMENT==2" NEWLINE "#include \"fsdata_alignment.h\"" NEWLINE "#endif" NEWLINE);
+#endif

  sprintf(lastFileVar, "NULL");

@@ -257,7 +344,14 @@ int main(int argc, char *argv[])
    printf("Warning: failed to delete fshdr.tmp\n");
  }

-  printf(NEWLINE "Processed %d files - done." NEWLINE NEWLINE, filesProcessed);
+  printf(NEWLINE "Processed %d files - done." NEWLINE, filesProcessed);
+#if MAKEFS_SUPPORT_DEFLATE
+  if (deflateNonSsiFiles) {
+    printf("(Deflated total byte reduction: %d bytes -> %d bytes (%.02f%%)" NEWLINE,
+      (int)overallDataBytes, (int)deflatedBytesReduced, (float)((deflatedBytesReduced*100.0)/overallDataBytes));
+  }
+#endif
+  printf(NEWLINE);

  while (first_file != NULL) {
     struct file_entry* fe = first_file;
@@ -295,16 +389,18 @@ static void copy_file(const char *filename_in, FILE *fout)
 {
  FILE *fin;
  size_t len;
+  void* buf;
  fin = fopen(filename_in, "rb");
  if (fin == NULL) {
    printf("Failed to open file \"%s\"\n", filename_in);
    exit(-1);
  }
-
-  while((len = fread(file_buffer_raw, 1, COPY_BUFSIZE, fin)) > 0)
+  buf = malloc(COPY_BUFSIZE);
+  while((len = fread(buf, 1, COPY_BUFSIZE, fin)) > 0)
  {
-    fwrite(file_buffer_raw, 1, len, fout);
+    fwrite(buf, 1, len, fout);
  }
+  free(buf);
  fclose(fin);
 }

@@ -347,7 +443,7 @@ int process_sub(FILE *data_file, FILE *struct_file)
           strncat(curSubdir, "/", freelen);
           strncat(curSubdir, curName, freelen - 1);
           curSubdir[sizeof(curSubdir) - 1] = 0;
-           printf(NEWLINE "processing subdirectory %s/..." NEWLINE, curSubdir);
+           printf("processing subdirectory %s/..." NEWLINE, curSubdir);
           filesProcessed += process_sub(data_file, struct_file);
           CHDIR("..");
           curSubdir[sublen] = 0;
@@ -376,58 +472,134 @@ int process_sub(FILE *data_file, FILE *struct_file)
  return filesProcessed;
 }

-int get_file_size(const char* filename)
+u8_t* get_file_data(const char* filename, int* file_size, int can_be_compressed, int* is_compressed)
 {
  FILE *inFile;
-  int file_size = -1;
+  size_t fsize = 0;
+  u8_t* buf;
+  size_t r;
+  int rs;
  inFile = fopen(filename, "rb");
  if (inFile == NULL) {
    printf("Failed to open file \"%s\"\n", filename);
    exit(-1);
  }
  fseek(inFile, 0, SEEK_END);
-  file_size = ftell(inFile);
-  fclose(inFile);
-  return file_size;
-}
-
-void process_file_data(const char *filename, FILE *data_file)
-{
-  FILE *source_file;
-  size_t len, written, i, src_off=0;
-
-  source_file = fopen(filename, "rb");
-  if (source_file == NULL) {
-    printf("Failed to open file \"%s\"\n", filename);
-    exit(-1);
+  rs = ftell(inFile);
+  if(rs < 0)
+  {
+     printf("ftell failed with %d\n", errno);
+     exit(-1);
  }
-
-  do {
-    size_t off = 0;
-    len = fread(file_buffer_raw, 1, COPY_BUFSIZE, source_file);
-    if (len > 0) {
-      for (i = 0; i < len; i++) {
-        sprintf(&file_buffer_c[off], "0x%02.2x,", file_buffer_raw[i]);
-        off += 5;
-        if ((++src_off % HEX_BYTES_PER_LINE) == 0) {
-          memcpy(&file_buffer_c[off], NEWLINE, NEWLINE_LEN);
-          off += NEWLINE_LEN;
+  fsize = (size_t)rs;
+  fseek(inFile, 0, SEEK_SET);
+  buf = (u8_t*)malloc(fsize);
+  LWIP_ASSERT("buf != NULL", buf != NULL);
+  r = fread(buf, 1, fsize, inFile);
+  *file_size = fsize;
+  *is_compressed = 0;
+#if MAKEFS_SUPPORT_DEFLATE
+  overallDataBytes += fsize;
+  if (deflateNonSsiFiles) {
+    if (can_be_compressed) {
+      if (fsize < OUT_BUF_SIZE) {
+        u8_t* ret_buf;
+        tdefl_status status;
+        size_t in_bytes = fsize;
+        size_t out_bytes = OUT_BUF_SIZE;
+        const void *next_in = buf;
+        void *next_out = s_outbuf;
+        /* create tdefl() compatible flags (we have to compose the low-level flags ourselves, or use tdefl_create_comp_flags_from_zip_params() but that means MINIZ_NO_ZLIB_APIS can't be defined). */
+        mz_uint comp_flags = s_tdefl_num_probes[MZ_MIN(10, deflate_level)] | ((deflate_level <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0);
+        if (!deflate_level) {
+          comp_flags |= TDEFL_FORCE_ALL_RAW_BLOCKS;
        }
+        status = tdefl_init(&g_deflator, NULL, NULL, comp_flags);
+        if (status != TDEFL_STATUS_OKAY) {
+          printf("tdefl_init() failed!\n");
+          exit(-1);
+        }
+        memset(s_outbuf, 0, sizeof(s_outbuf));
+        status = tdefl_compress(&g_deflator, next_in, &in_bytes, next_out, &out_bytes, TDEFL_FINISH);
+        if (status != TDEFL_STATUS_DONE) {
+          printf("deflate failed: %d\n", status);
+          exit(-1);
+        }
+        LWIP_ASSERT("out_bytes <= COPY_BUFSIZE", out_bytes <= OUT_BUF_SIZE);
+        if (out_bytes < fsize) {
+          ret_buf = (u8_t*)malloc(out_bytes);
+          LWIP_ASSERT("ret_buf != NULL", ret_buf != NULL);
+          memcpy(ret_buf, s_outbuf, out_bytes);
+          {
+            /* sanity-check compression be inflating and comparing to the original */
+            tinfl_status dec_status;
+            tinfl_decompressor inflator;
+            size_t dec_in_bytes = out_bytes;
+            size_t dec_out_bytes = OUT_BUF_SIZE;
+            next_out = s_checkbuf;
+
+            tinfl_init(&inflator);
+            memset(s_checkbuf, 0, sizeof(s_checkbuf));
+            dec_status = tinfl_decompress(&inflator, (const mz_uint8 *)ret_buf, &dec_in_bytes, s_checkbuf, (mz_uint8 *)next_out, &dec_out_bytes, 0);
+            LWIP_ASSERT("tinfl_decompress failed", dec_status == TINFL_STATUS_DONE);
+            LWIP_ASSERT("tinfl_decompress size mismatch", fsize == dec_out_bytes);
+            LWIP_ASSERT("decompressed memcmp failed", !memcmp(s_checkbuf, buf, fsize));
+          }
+          /* free original buffer, use compressed data + size */
+          free(buf);
+          buf = ret_buf;
+          *file_size = out_bytes;
+          printf(" - deflate: %d bytes -> %d bytes (%.02f%%)" NEWLINE, (int)fsize, (int)out_bytes, (float)((out_bytes*100.0)/fsize));
+          deflatedBytesReduced += (size_t)(fsize - out_bytes);
+          *is_compressed = 1;
+        } else {
+          printf(" - uncompressed: (would be %d bytes larger using deflate)" NEWLINE, (int)(out_bytes - fsize));
+        }
+      } else {
+        printf(" - uncompressed: (file is larger than deflate bufer)" NEWLINE);
      }
-      written = fwrite(file_buffer_c, 1, off, data_file);
+    } else {
+      printf(" - SSI file, cannot be compressed" NEWLINE);
    }
-  } while(len > 0);
-  fclose(source_file);
+  }
+#else
+  LWIP_UNUSED_ARG(compress);
+#endif
+  fclose(inFile);
+  return buf;
 }

-int write_checksums(FILE *struct_file, const char *filename, const char *varname,
-                    u16_t hdr_len, u16_t hdr_chksum)
+void process_file_data(FILE* data_file, u8_t* file_data, size_t file_size)
+{
+  size_t written, i, src_off=0;
+
+  size_t off = 0;
+  for (i = 0; i < file_size; i++) {
+    LWIP_ASSERT("file_buffer_c overflow", off < sizeof(file_buffer_c) - 5);
+    sprintf(&file_buffer_c[off], "0x%02.2x,", file_data[i]);
+    off += 5;
+    if ((++src_off % HEX_BYTES_PER_LINE) == 0) {
+      LWIP_ASSERT("file_buffer_c overflow", off < sizeof(file_buffer_c) - NEWLINE_LEN);
+      memcpy(&file_buffer_c[off], NEWLINE, NEWLINE_LEN);
+      off += NEWLINE_LEN;
+    }
+    if (off + 20 >= sizeof(file_buffer_c)) {
+      written = fwrite(file_buffer_c, 1, off, data_file);
+      LWIP_ASSERT("written == off", written == off);
+      off = 0;
+    }
+  }
+  written = fwrite(file_buffer_c, 1, off, data_file);
+  LWIP_ASSERT("written == off", written == off);
+}
+
+int write_checksums(FILE *struct_file, const char *varname,
+                    u16_t hdr_len, u16_t hdr_chksum, const u8_t* file_data, size_t file_size)
 {
  int chunk_size = TCP_MSS;
-  int offset;
+  int offset, src_offset;
  size_t len;
  int i = 0;
-  FILE *f;
 #if LWIP_TCP_TIMESTAMPS
  /* when timestamps are used, usable space is 12 bytes less per segment */
  chunk_size -= 12;
@@ -436,29 +608,24 @@ int write_checksums(FILE *struct_file, const char *filename, const char *varname
  fprintf(struct_file, "#if HTTPD_PRECALCULATED_CHECKSUM" NEWLINE);
  fprintf(struct_file, "const struct fsdata_chksum chksums_%s[] = {" NEWLINE, varname);

-  memset(file_buffer_raw, 0xab, sizeof(file_buffer_raw));
-  f = fopen(filename, "rb");
-  if (f == NULL) {
-    printf("Failed to open file \"%s\"\n", filename);
-    exit(-1);
-  }
  if (hdr_len > 0) {
    /* add checksum for HTTP header */
    fprintf(struct_file, "{%d, 0x%04x, %d}," NEWLINE, 0, hdr_chksum, hdr_len);
    i++;
  }
+  src_offset = 0;
  for (offset = hdr_len; ; offset += len) {
    unsigned short chksum;
-    len = fread(file_buffer_raw, 1, chunk_size, f);
+    void* data = (void*)&file_data[src_offset];
+    len = LWIP_MIN(chunk_size, (int)file_size - src_offset);
    if (len == 0) {
      break;
    }
-    chksum = ~inet_chksum(file_buffer_raw, (u16_t)len);
+    chksum = ~inet_chksum(data, (u16_t)len);
    /* add checksum for data */
    fprintf(struct_file, "{%d, 0x%04x, %d}," NEWLINE, offset, chksum, len);
    i++;
  }
-  fclose(f);
  fprintf(struct_file, "};" NEWLINE);
  fprintf(struct_file, "#endif /* HTTPD_PRECALCULATED_CHECKSUM */" NEWLINE);
  return i;
@@ -480,7 +647,7 @@ static void fix_filename_for_c(char* qualifiedName, size_t max_len)
 {
   struct file_entry* f;
   size_t len = strlen(qualifiedName);
-   char *new_name = malloc(len + 2);
+   char *new_name = (char*)malloc(len + 2);
   int filename_ok;
   int cnt = 0;
   size_t i;
@@ -516,7 +683,7 @@ static void fix_filename_for_c(char* qualifiedName, size_t max_len)

 static void register_filename(const char* qualifiedName)
 {
-   struct file_entry* fe = malloc(sizeof(struct file_entry));
+   struct file_entry* fe = (struct file_entry*)malloc(sizeof(struct file_entry));
   fe->filename_c = strdup(qualifiedName);
   fe->next = NULL;
   if (first_file == NULL) {
@@ -527,6 +694,17 @@ static void register_filename(const char* qualifiedName)
   }
 }

+int is_ssi_file(const char* filename)
+{
+  size_t loop;
+  for (loop = 0; loop < NUM_SHTML_EXTENSIONS; loop++) {
+    if (strstr(filename, g_pcSSIExtensions[loop])) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
 int process_file(FILE *data_file, FILE *struct_file, const char *filename)
 {
  char varname[MAX_PATH_LEN];
@@ -536,6 +714,11 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
  u16_t http_hdr_chksum = 0;
  u16_t http_hdr_len = 0;
  int chksum_count = 0;
+  u8_t flags = 0;
+  const char* flags_str;
+  u8_t has_content_len;
+  u8_t* file_data;
+  int is_compressed = 0;

  /* create qualified name (TODO: prepend slash or not?) */
  sprintf(qualifiedName,"%s/%s", curSubdir, filename);
@@ -545,10 +728,12 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
  fix_filename_for_c(varname, MAX_PATH_LEN);
  register_filename(varname);
 #if ALIGN_PAYLOAD
-  /* to force even alignment of array */
+  /* to force even alignment of array, type 1 */
+  fprintf(data_file, "#if FSDATA_FILE_ALIGNMENT==1" NEWLINE);
  fprintf(data_file, "static const " PAYLOAD_ALIGN_TYPE " dummy_align_%s = %d;" NEWLINE, varname, payload_alingment_dummy_counter++);
+  fprintf(data_file, "#endif" NEWLINE);
 #endif /* ALIGN_PAYLOAD */
-  fprintf(data_file, "static const unsigned char data_%s[] = {" NEWLINE, varname);
+  fprintf(data_file, "static const unsigned char FSDATA_ALIGN_PRE data_%s[] FSDATA_ALIGN_POST = {" NEWLINE, varname);
  /* encode source file name (used by file system, not returned to browser) */
  fprintf(data_file, "/* %s (%d chars) */" NEWLINE, qualifiedName, strlen(qualifiedName)+1);
  file_put_ascii(data_file, qualifiedName, strlen(qualifiedName)+1, &i);
@@ -561,12 +746,17 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
 #endif /* ALIGN_PAYLOAD */
  fprintf(data_file, NEWLINE);

-  file_size = get_file_size(filename);
+  has_content_len = !is_ssi_file(filename);
+  file_data = get_file_data(filename, &file_size, includeHttpHeader && has_content_len, &is_compressed);
  if (includeHttpHeader) {
-    file_write_http_header(data_file, filename, file_size, &http_hdr_len, &http_hdr_chksum);
+    file_write_http_header(data_file, filename, file_size, &http_hdr_len, &http_hdr_chksum, has_content_len, is_compressed);
+    flags = FS_FILE_FLAGS_HEADER_INCLUDED;
+    if (has_content_len) {
+      flags |= FS_FILE_FLAGS_HEADER_PERSISTENT;
+    }
  }
  if (precalcChksum) {
-    chksum_count = write_checksums(struct_file, filename, varname, http_hdr_len, http_hdr_chksum);
+    chksum_count = write_checksums(struct_file, varname, http_hdr_len, http_hdr_chksum, file_data, file_size);
  }

  /* build declaration of struct fsdata_file in temp file */
@@ -575,7 +765,22 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
  fprintf(struct_file, "data_%s," NEWLINE, varname);
  fprintf(struct_file, "data_%s + %d," NEWLINE, varname, i);
  fprintf(struct_file, "sizeof(data_%s) - %d," NEWLINE, varname, i);
-  fprintf(struct_file, "%d," NEWLINE, includeHttpHeader);
+  switch(flags)
+  {
+  case(FS_FILE_FLAGS_HEADER_INCLUDED):
+     flags_str = "FS_FILE_FLAGS_HEADER_INCLUDED";
+     break;
+  case(FS_FILE_FLAGS_HEADER_PERSISTENT):
+     flags_str = "FS_FILE_FLAGS_HEADER_PERSISTENT";
+     break;
+  case(FS_FILE_FLAGS_HEADER_INCLUDED | FS_FILE_FLAGS_HEADER_PERSISTENT):
+     flags_str = "FS_FILE_FLAGS_HEADER_INCLUDED | FS_FILE_FLAGS_HEADER_PERSISTENT";
+     break;
+  default:
+     flags_str = "0";
+     break;
+  }
+  fprintf(struct_file, "%s," NEWLINE, flags_str);
  if (precalcChksum) {
    fprintf(struct_file, "#if HTTPD_PRECALCULATED_CHECKSUM" NEWLINE);
    fprintf(struct_file, "%d, chksums_%s," NEWLINE, chksum_count, varname);
@@ -587,14 +792,14 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
  /* write actual file contents */
  i = 0;
  fprintf(data_file, NEWLINE "/* raw file data (%d bytes) */" NEWLINE, file_size);
-  process_file_data(filename, data_file);
+  process_file_data(data_file, file_data, file_size);
  fprintf(data_file, "};" NEWLINE NEWLINE);
-
+  free(file_data);
  return 0;
 }

-int file_write_http_header(FILE *data_file, const char *filename, int file_size,
-                           u16_t *http_hdr_len, u16_t *http_hdr_chksum)
+int file_write_http_header(FILE *data_file, const char *filename, int file_size, u16_t *http_hdr_len,
+                           u16_t *http_hdr_chksum, u8_t provide_content_len, int is_compressed)
 {
  int i = 0;
  int response_type = HTTP_HDR_OK;
@@ -606,17 +811,7 @@ int file_write_http_header(FILE *data_file, const char *filename, int file_size,
  u16_t acc;
  const char *file_ext;
  int j;
-  u8_t keepalive = useHttp11;
-
-  if (keepalive) {
-    size_t loop;
-    for (loop = 0; loop < NUM_SHTML_EXTENSIONS; loop++) {
-      if (strstr(filename, g_pcSSIExtensions[loop])) {
-        /* no keepalive connection for SSI files */
-        keepalive = 0;
-      }
-    }
-  }
+  u8_t provide_last_modified = includeLastModified;

  memset(hdr_buf, 0, sizeof(hdr_buf));
  
@@ -685,38 +880,78 @@ int file_write_http_header(FILE *data_file, const char *filename, int file_size,
    }
  }

-  if (useHttp11) {
+  /* Content-Length is used for persistent connections in HTTP/1.1 but also for
+     download progress in older versions
+     @todo: just use a big-enough buffer and let the HTTPD send spaces? */
+  if (provide_content_len) {
    char intbuf[MAX_PATH_LEN];
    int content_len = file_size;
    memset(intbuf, 0, sizeof(intbuf));
-
-    if (!keepalive) {
-      content_len *= 2;
+    cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONTENT_LENGTH];
+    cur_len = strlen(cur_string);
+    fprintf(data_file, NEWLINE "/* \"%s%d\r\n\" (%d+ bytes) */" NEWLINE, cur_string, content_len, cur_len+2);
+    written += file_put_ascii(data_file, cur_string, cur_len, &i);
+    if (precalcChksum) {
+      memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
+      hdr_len += cur_len;
    }
+
+    _itoa(content_len, intbuf, 10);
+    strcat(intbuf, "\r\n");
+    cur_len = strlen(intbuf);
+    written += file_put_ascii(data_file, intbuf, cur_len, &i);
+    i = 0;
+    if (precalcChksum) {
+      memcpy(&hdr_buf[hdr_len], intbuf, cur_len);
+      hdr_len += cur_len;
+    }
+  }
+  if (provide_last_modified) {
+    char modbuf[256];
+    struct stat stat_data;
+    struct tm* t;
+    memset(modbuf, 0, sizeof(modbuf));
+    memset(&stat_data, 0, sizeof(stat_data));
+    cur_string = modbuf;
+    strcpy(modbuf, "Last-Modified: ");
+    if(stat(filename, &stat_data) != 0)
    {
-      cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONTENT_LENGTH];
-      cur_len = strlen(cur_string);
-      fprintf(data_file, NEWLINE "/* \"%s%d\r\n\" (%d+ bytes) */" NEWLINE, cur_string, content_len, cur_len+2);
-      written += file_put_ascii(data_file, cur_string, cur_len, &i);
-      if (precalcChksum) {
-        memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
-        hdr_len += cur_len;
-      }
-
-      _itoa(content_len, intbuf, 10);
-      strcat(intbuf, "\r\n");
-      cur_len = strlen(intbuf);
-      written += file_put_ascii(data_file, intbuf, cur_len, &i);
-      i = 0;
-      if (precalcChksum) {
-        memcpy(&hdr_buf[hdr_len], intbuf, cur_len);
-        hdr_len += cur_len;
-      }
+       printf("stat(%s) failed with error %d\n", filename, errno);
+       exit(-1);
+    }
+    t = gmtime(&stat_data.st_mtime);
+    if(t == NULL)
+    {
+       printf("gmtime() failed with error %d\n", errno);
+       exit(-1);
+    }
+    strftime(&modbuf[15], sizeof(modbuf)-15, "%a, %d %b %Y %H:%M:%S GMT", t);
+    cur_len = strlen(cur_string);
+    fprintf(data_file, NEWLINE "/* \"%s\"\r\n\" (%d+ bytes) */" NEWLINE, cur_string, cur_len+2);
+    written += file_put_ascii(data_file, cur_string, cur_len, &i);
+    if (precalcChksum) {
+      memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
+      hdr_len += cur_len;
    }

-    if (keepalive) {
+    modbuf[0] = 0;
+    strcat(modbuf, "\r\n");
+    cur_len = strlen(modbuf);
+    written += file_put_ascii(data_file, modbuf, cur_len, &i);
+    i = 0;
+    if (precalcChksum) {
+      memcpy(&hdr_buf[hdr_len], modbuf, cur_len);
+      hdr_len += cur_len;
+    }
+  }
+
+  /* HTTP/1.1 implements persistent connections */
+  if (useHttp11) {
+    if (provide_content_len) {
      cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONN_KEEPALIVE];
    } else {
+      /* no Content-Length available, so a persistent connection is no possible
+         because the client does not know the data length */
      cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONN_CLOSE];
    }
    cur_len = strlen(cur_string);
@@ -729,11 +964,29 @@ int file_write_http_header(FILE *data_file, const char *filename, int file_size,
    }
  }

+#if MAKEFS_SUPPORT_DEFLATE
+  if (is_compressed) {
+    /* tell the client about the deflate encoding */
+    LWIP_ASSERT("error", deflateNonSsiFiles);
+    cur_string = "Content-Encoding: deflate\r\n";
+    cur_len = strlen(cur_string);
+    fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
+    written += file_put_ascii(data_file, cur_string, cur_len, &i);
+    i = 0;
+  }
+#else
+  LWIP_UNUSED_ARG(is_compressed);
+#endif
+
+  /* write content-type, ATTENTION: this includes the double-CRLF! */
  cur_string = file_type;
  cur_len = strlen(cur_string);
  fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
  written += file_put_ascii(data_file, cur_string, cur_len, &i);
  i = 0;
+
+  /* ATTENTION: headers are done now (double-CRLF has been written!) */
+
  if (precalcChksum) {
    memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
    hdr_len += cur_len;