[setup - the official Cygwin setup program] branch master, updated. release_2.893-14-gf318f27

[jturney@sourceware.org]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 18517 bytes --]




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=f318f2722d5b0c82779a7240f65a6bbc2db5d58f

commit f318f2722d5b0c82779a7240f65a6bbc2db5d58f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Oct 13 17:16:40 2018 +0100

    Add zstd package to .appveyor.yml
    
    Reformat so we can escape newlines to avoid an absurdly long line

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=ddb9b0952a4824079644dbd1afe84b4ea2ae8b97

commit ddb9b0952a4824079644dbd1afe84b4ea2ae8b97
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Oct 13 14:23:00 2018 +0100

    Add zstd packages to build instructions
    
    Also adjust for 'if building from git' is the only way, since we don't make
    source releases.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=c96417522f9cb58a46b2c9fbac59a43926aa28de

commit c96417522f9cb58a46b2c9fbac59a43926aa28de
Author: Achim Gratz <Stromeko@Stromeko.DE>
Date:   Sun Sep 2 18:41:24 2018 +0200

    Add support for ZStandard compression
    
    Add support for ZStandard compression for packages and .ini files

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=60912117f46f07439582920a2c24d3b84296fdd7

commit 60912117f46f07439582920a2c24d3b84296fdd7
Author: Achim Gratz <Stromeko@Stromeko.DE>
Date:   Wed Oct 10 19:44:56 2018 +0200

    Remove misleading copy&paste comments

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/setup.git;h=31cdecc78e525a90effca1e3b86b7632609b2091

commit 31cdecc78e525a90effca1e3b86b7632609b2091
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun Oct 7 22:41:06 2018 +0100

    Avoid stringop-overflow warning with gcc8
    
    desktop.cc: In function 'void start_menu(const string&, const string&, const string&, const string&)':
    desktop.cc:110:11: error: 'char* strncat(char*, const char*, size_t)' specified bound 260 equals destination size [-Werror=stringop-overflow=]
    
    I think strlcat() was meant here, which MinGW doesn't have.  In it's
    absence, open-code it's equivalent.
    
    (SHGetSpecialFolderLocation() returns a pathname of length at most MAX_PATH,
    and make_link() is limited to accepting a pathname of length MAX_PATH, so we
    want to append our folder name, while truncating the result to MAX_PATH.)


Diff:
---
 .appveyor.yml    |   44 ++++++---
 Makefile.am      |    5 +-
 README           |   35 ++++----
 compress.cc      |   13 +++-
 compress_gz.h    |    4 -
 compress_xz.h    |    3 -
 compress_zstd.cc |  257 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 compress_zstd.h  |   65 ++++++++++++++
 configure.ac     |    1 +
 desktop.cc       |    2 +-
 ini.h            |    2 +-
 11 files changed, 389 insertions(+), 42 deletions(-)

diff --git a/.appveyor.yml b/.appveyor.yml
index b007e05..9998cc6 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -7,22 +7,36 @@ environment:
   - HOST: i686-w64-mingw32
   - HOST: x86_64-w64-mingw32
 install:
-- cmd: |-
-    if "%HOST%"=="i686-w64-mingw32" set PKGARCH="mingw64-i686"
-    if "%HOST%"=="x86_64-w64-mingw32" set PKGARCH="mingw64-x86_64"
-    if NOT DEFINED PKGARCH exit 1
-    set SETUP=setup-x86.exe
-    echo Updating Cygwin and installing build prerequisites
-    %CYGWIN_ROOT%\%SETUP% -qnNdO -R "%CYGWIN_ROOT%" -s "%CYGWIN_MIRROR%" -l "%CYGWIN_CACHE%" -g -P "autoconf,automake,bison,flex,libtool,make,%PKGARCH%-headers,%PKGARCH%-gcc-g++,%PKGARCH%-libgcrypt,%PKGARCH%-libsolv,%PKGARCH%-bzip2,%PKGARCH%-xz,%PKGARCH%-zlib,pkg-config,upx"
-cache: C:\cache
+ - if "%HOST%"=="i686-w64-mingw32" set PKGARCH="mingw64-i686"
+ - if "%HOST%"=="x86_64-w64-mingw32" set PKGARCH="mingw64-x86_64"
+ - if NOT DEFINED PKGARCH exit 1
+ - set SETUP=setup-x86.exe
+ - echo Updating Cygwin and installing build prerequisites
+ - "%CYGWIN_ROOT%\\%SETUP% -qnNdO -R %CYGWIN_ROOT% -s %CYGWIN_MIRROR% -l %CYGWIN_CACHE% -g -P \
+autoconf,\
+automake,\
+bison,\
+flex,\
+libtool,\
+make,\
+%PKGARCH%-bzip2,\
+%PKGARCH%-gcc-g++,\
+%PKGARCH%-headers,\
+%PKGARCH%-libgcrypt,\
+%PKGARCH%-libsolv,\
+%PKGARCH%-xz,\
+%PKGARCH%-zlib,\
+%PKGARCH%-zstd,\
+pkg-config,\
+upx"
 build_script:
-- cmd: |-
-    echo Bootstrap running...
-    %CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./bootstrap.sh --host=%HOST%"
-    echo Make running...
-    %CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make"
-    echo Strip/UPX running...
-    %CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make strip upx"
+ - echo Bootstrap running...
+ - '%CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./bootstrap.sh --host=%HOST%"'
+ - echo Make running...
+ - '%CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make"'
+ - echo Strip/UPX running...
+ - '%CYGWIN_ROOT%/bin/bash -lc "cd $APPVEYOR_BUILD_FOLDER; make strip upx"'
+cache: C:\cache
 test: off
 deploy: off
 artifacts:
diff --git a/Makefile.am b/Makefile.am
index a4f9a12..0ad4e5c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -29,7 +29,7 @@ AM_YFLAGS = -d
 AM_LFLAGS = -8
 WINDRES = @WINDRES@
 AM_CPPFLAGS = -DLZMA_API_STATIC -I$(srcdir)/libgetopt++/include \
-	       $(ZLIB_CFLAGS) $(LZMA_CFLAGS) $(LIBCRYPT_CFLAGS) $(LIBSOLV_CFLAGS)
+	       $(ZLIB_CFLAGS) $(LZMA_CFLAGS) $(ZSTD_CFLAGS) $(LIBCRYPT_CFLAGS) $(LIBSOLV_CFLAGS)
 
 inilex_CXXFLAGS:=-Wno-sign-compare
 
@@ -100,6 +100,7 @@ inilint_SOURCES = \
 @SETUP@_LDADD = \
 	libgetopt++/libgetopt++.la \
 	$(LIBGCRYPT_LIBS) \
+	$(ZSTD_LIBS) \
 	$(LZMA_LIBS) \
 	$(BZ2_LIBS) \
 	$(ZLIB_LIBS) \
@@ -124,6 +125,8 @@ inilint_SOURCES = \
 	compress_gz.h \
 	compress_xz.cc \
 	compress_xz.h \
+	compress_zstd.cc \
+	compress_zstd.h \
 	confirm.cc \
 	confirm.h \
 	ConnectionSetting.cc \
diff --git a/README b/README
index fa2b170..b8f3f7c 100644
--- a/README
+++ b/README
@@ -10,26 +10,24 @@ Cygwin
 Setup should build out-of-the-box on any Cygwin environment that has all the
 required packages and their dependencies installed:
 
+  - autoconf
+  - automake
+  - bison
+  - flex
+  - libtool
   - make
-  - mingw64-${arch}-headers
+  - mingw64-${arch}-bzip2
   - mingw64-${arch}-gcc-g++
+  - mingw64-${arch}-headers
   - mingw64-${arch}-libgcrypt
   - mingw64-${arch}-libsolv
-  - mingw64-${arch}-bzip2
   - mingw64-${arch}-xz
   - mingw64-${arch}-zlib
+  - mingw64-${arch}-zstd
   - upx (optional)
 
 The ${arch} needs to be replaced with either "i686" or "x86_64"
-depending on the target architecture to build for.  The following
-additional packages are required if building from Git, or if you want
-to make changes to the build system.
-
-  - autoconf
-  - automake
-  - libtool
-  - flex
-  - bison
+depending on the target architecture to build for.
 
 Fedora
 ------
@@ -37,26 +35,31 @@ Fedora
 Setup should also build out-of-the-box in a Fedora environment that has all the
 required packages and their dependencies installed:
 
-  - make
+  - automake
+  - bison
+  - flex
   - libtool
+  - make
+  - mingw${arch}-bzip2-static
   - mingw${arch}-gcc-c++
-  - mingw${arch}-zlib-static
   - mingw${arch}-libgcrypt-static
   - mingw${arch}-libgnurx-static
   - mingw${arch}-libsolv-static (*)
-  - mingw${arch}-bzip2-static
-  - mingw${arch}-xz-libs-static
+  - mingw${arch}-libzstd-static (**)
   - mingw${arch}-winpthreads-static
+  - mingw${arch}-xz-libs-static
+  - mingw${arch}-zlib-static
   - upx (optional)
 
 The ${arch} needs to be replaced with either "32" or "64"
 depending on the target architecture to build for.
 
 (*) Requires 'dnf copr enable jturney/mingw-libsolv'
+(**) Requires 'dnf copr enable jturney/mingw-zstd'
 
 Build commands:
 
-0) If building from git, obtain this project's code:
+0) Obtain this project's source code:
    $ git clone git://sourceware.org/git/cygwin-apps/setup.git
    $ cd setup
 
diff --git a/compress.cc b/compress.cc
index 7052f96..9ff41d3 100644
--- a/compress.cc
+++ b/compress.cc
@@ -17,6 +17,7 @@
 #include "compress_gz.h"
 #include "compress_bz.h"
 #include "compress_xz.h"
+#include "compress_zstd.h"
 #include <string.h>
 
 /* In case you are wondering why the file magic is not in one place:
@@ -28,7 +29,7 @@
  * the class could test itself. 
  */
 
-#define longest_magic 14 /* lzma_alone */
+#define longest_magic 18 /* ZStandard longest frame header (magic is only 4 bytes) */
 
 io_stream *
 compress::decompress (io_stream * original)
@@ -49,6 +50,16 @@ compress::decompress (io_stream * original)
 	  delete rv;
 	  return NULL;
 	}
+      else if (compress_zstd::is_zstd (magic, 18))
+	{
+	  compress_zstd *rv = new compress_zstd (original);
+	  if (!rv->error ())
+	    return rv;
+	  /* else */
+	  rv->release_original();
+	  delete rv;
+	  return NULL;
+	}
       else if (memcmp (magic, "BZh", 3) == 0)
 	{
 	  compress_bz *rv = new compress_bz (original);
diff --git a/compress_gz.h b/compress_gz.h
index bbed852..50b6e66 100644
--- a/compress_gz.h
+++ b/compress_gz.h
@@ -16,10 +16,6 @@
 #ifndef SETUP_COMPRESS_GZ_H
 #define SETUP_COMPRESS_GZ_H
 
-/* this is the parent class for all compress IO operations. 
- * It 
- */
-
 #include "compress.h"
 #include <zlib.h>
 
diff --git a/compress_xz.h b/compress_xz.h
index 07572ef..24cbb09 100644
--- a/compress_xz.h
+++ b/compress_xz.h
@@ -16,9 +16,6 @@
 #ifndef SETUP_COMPRESS_XZ_H
 #define SETUP_COMPRESS_XZ_H
 
-/* this is the parent class for all compress IO operations. 
- */
-
 #include "compress.h"
 #include <lzma.h>
 
diff --git a/compress_zstd.cc b/compress_zstd.cc
new file mode 100644
index 0000000..3588dbd
--- /dev/null
+++ b/compress_zstd.cc
@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2018, Cygwin
+ *
+ *     This program is free software; you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation; either version 2 of the License, or
+ *     (at your option) any later version.
+ *
+ *     A copy of the GNU General Public License can be found at
+ *     http://www.gnu.org/
+ *
+ */
+
+#include "compress_zstd.h"
+
+#include <stdexcept>
+
+#include <errno.h>
+#include <memory.h>
+#include <malloc.h>
+
+/*
+ * Predicate: the stream is open for read.
+ */
+compress_zstd::compress_zstd (io_stream * parent)
+:
+  original(NULL),
+  owns_original(true),
+  lasterr(0)
+{
+  /* read only */
+  if (!parent || parent->error())
+    {
+      lasterr = EBADF;
+      return;
+    }
+  original = parent;
+
+  state = (struct private_data *)calloc(sizeof(*state), 1);
+  if (state == NULL)
+    {
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+
+  state->stream = ZSTD_createDStream();
+  if (state->stream == NULL)
+    {
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+  ZSTD_initDStream(state->stream);
+  state->out_block.size = state->out_block.pos  = state->out_pos = state->out_bsz = ZSTD_DStreamOutSize();
+  state->out_block.dst  = (unsigned char *)malloc(state->out_bsz);
+  if (state->out_block.dst == NULL)
+    {
+      free(state->out_block.dst);
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+  state->in_block.size = state->in_block.pos  = state->in_bsz = ZSTD_DStreamInSize();
+  state->in_block.src  = (unsigned char *)malloc(state->in_bsz);
+  state->in_pos  = 0;
+  if (state->in_block.src == NULL)
+    {
+      free(state->out_block.dst);
+      free((void*)state->in_block.src);
+      free(state);
+      lasterr = ENOMEM;
+      return;
+    }
+}
+
+ssize_t
+compress_zstd::read (void *buffer, size_t len)
+{
+  /* there is no recovery from a busted stream */
+  if (this->lasterr)
+    {
+      return -1;
+    }
+  if (len == 0)
+    {
+      return 0;
+    }
+
+  size_t lenRemaining = len;
+  size_t lenBuffered  = 0;
+  do
+    {
+      if (state->in_block.size > 0 && state->in_block.pos >= state->in_block.size)
+        {
+	  /* no compressed data ready; read some more input */
+	  state->in_block.size = state->in_bsz;
+	  ssize_t got = this->original->read((void *)state->in_block.src, state->in_bsz);
+	  if (got >= 0)
+	    {
+	      state->in_block.size = got;
+	      state->in_block.pos = 0;
+	    }
+	  else
+	    {
+	      lasterr = EIO;
+	      return -1;
+	    }
+	  continue;
+        }
+
+      if (state->out_pos < state->out_block.pos)
+	{
+	  /* output buffer has unused data */
+	  ssize_t tmplen = std::min (state->out_block.pos - state->out_pos, lenRemaining);
+	  memcpy (&((char *)buffer)[lenBuffered], &((char *)state->out_block.dst)[state->out_pos], tmplen);
+	  state->out_pos += tmplen;
+	  lenBuffered    += tmplen;
+	  lenRemaining   -= tmplen;
+	  if (state->eof)
+	    {
+	      break;
+	    }
+	}
+      else
+	{
+	  if (state->eof)
+	    {
+	      break;
+	    }
+	  /* output buffer is empty; decompress more data */
+	  state->out_block.size = state->out_bsz;
+	  state->out_pos = state->out_block.pos = 0;
+	  size_t ret = ZSTD_decompressStream (state->stream, &state->out_block, &state->in_block);
+	  if (ZSTD_isError(ret))
+	    {
+	      // TODO return/print error
+	      return -1;
+	    }
+	  state->eof = (ret == 0);
+	}
+    }
+  while (lenRemaining != 0);
+
+  return (len - lenRemaining);
+}
+
+ssize_t
+compress_zstd::write (const void *buffer, size_t len)
+{
+  throw new std::logic_error("compress_zstd::write is not implemented");
+}
+
+ssize_t
+compress_zstd::peek (void *buffer, size_t len)
+{
+  /* can only peek 512 bytes */
+  if (len > 512)
+    return ENOMEM;
+
+  // we only peek at the beginning of a file, so no buffer tearing can happen
+  // do a real read first…
+  ssize_t got = read (buffer, len);
+  if (got >= 0)
+    {
+      // …then rewind read position for the next read()
+      state->out_pos -= got;
+    }
+  /* error */
+  return got;
+}
+
+long
+compress_zstd::tell ()
+{
+  throw new std::logic_error("compress_zstd::tell is not implemented");
+}
+
+int
+compress_zstd::seek (long where, io_stream_seek_t whence)
+{
+  throw new std::logic_error("compress_zstd::seek is not implemented");
+}
+
+int
+compress_zstd::error ()
+{
+  return lasterr;
+}
+
+int
+compress_zstd::set_mtime (time_t mtime)
+{
+  if (original)
+    return original->set_mtime (mtime);
+  return 1;
+}
+
+time_t
+compress_zstd::get_mtime ()
+{
+  if (original)
+    return original->get_mtime ();
+  return 0;
+}
+
+mode_t
+compress_zstd::get_mode ()
+{
+  if (original)
+    return original->get_mode ();
+  return 0;
+}
+
+void
+compress_zstd::release_original ()
+{
+  owns_original = false;
+}
+
+void
+compress_zstd::destroy ()
+{
+  if (state)
+    {
+      ZSTD_freeDStream(state->stream);
+
+      if (state->out_block.dst)
+        {
+          free (state->out_block.dst);
+          state->out_block.dst = NULL;
+        }
+
+      if (state->in_block.src)
+        {
+          free ((void*)state->in_block.src);
+          state->in_block.src = NULL;
+        }
+
+      free(state);
+      state = NULL;
+    }
+
+  if (original && owns_original)
+    delete original;
+}
+
+compress_zstd::~compress_zstd ()
+{
+  destroy ();
+}
+
+bool
+compress_zstd::is_zstd (void * buffer, size_t len)
+{
+  return (ZSTD_getFrameContentSize(buffer, len) != ZSTD_CONTENTSIZE_ERROR);
+}
diff --git a/compress_zstd.h b/compress_zstd.h
new file mode 100644
index 0000000..be5712c
--- /dev/null
+++ b/compress_zstd.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018, Cygwin
+ *
+ *     This program is free software; you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation; either version 2 of the License, or
+ *     (at your option) any later version.
+ *
+ *     A copy of the GNU General Public License can be found at
+ *     http://www.gnu.org/
+ *
+ */
+
+#ifndef SETUP_COMPRESS_ZSTD_H
+#define SETUP_COMPRESS_ZSTD_H
+
+#include "compress.h"
+#include <zstd.h>
+
+class compress_zstd:public compress
+{
+public:
+  compress_zstd (io_stream *); /* decompress (read) only */
+  virtual ssize_t read (void *buffer, size_t len);
+  virtual ssize_t write (const void *buffer, size_t len); /* not implemented */
+  virtual ssize_t peek (void *buffer, size_t len);
+  virtual long tell (); /* not implemented */
+  virtual int seek (long where, io_stream_seek_t whence); /* not implemented */
+  virtual int error ();
+  virtual const char *next_file_name () { return NULL; };
+  virtual int set_mtime (time_t);
+  virtual time_t get_mtime ();
+  virtual mode_t get_mode ();
+  virtual size_t get_size () {return 0;};
+  virtual ~compress_zstd ();
+  static bool is_zstd (void *buffer, size_t len);
+  virtual void release_original(); /* give up ownership of original io_stream */
+
+private:
+  compress_zstd () {};
+
+  io_stream *original;
+  bool owns_original;
+  int lasterr;
+  void destroy ();
+
+  struct private_data {
+    ZSTD_DStream    *stream;
+    ZSTD_outBuffer   out_block;
+    size_t           out_bsz;
+    size_t           out_pos;
+    uint64_t         total_out;
+    char             eof; /* True = found end of compressed data. */
+    ZSTD_inBuffer    in_block;
+    size_t           in_bsz;
+    size_t           in_pos;
+    uint64_t         total_in;
+    size_t           in_processed;
+    size_t           out_processed;
+  };
+
+  struct private_data *state;
+};
+
+#endif /* SETUP_COMPRESS_ZSTD_H */
diff --git a/configure.ac b/configure.ac
index 3854088..08fe16b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -51,6 +51,7 @@ AC_CHECK_TOOL(OBJCOPY, objcopy, objcopy)
 dnl dependencies we can check for using pkgconfig
 PKG_CHECK_MODULES(ZLIB, [zlib])
 PKG_CHECK_MODULES(LZMA, [liblzma])
+PKG_CHECK_MODULES(ZSTD, [libzstd])
 PKG_CHECK_MODULES(LIBSOLV, [libsolv])
 
 dnl dependencies we need to check for by hand
diff --git a/desktop.cc b/desktop.cc
index 927c02f..eec8ca9 100644
--- a/desktop.cc
+++ b/desktop.cc
@@ -107,7 +107,7 @@ start_menu (const std::string& title, const std::string& target,
 			      issystem ? CSIDL_COMMON_PROGRAMS :
 			      CSIDL_PROGRAMS, &id);
   SHGetPathFromIDList (id, path);
-  strncat (path, "/Cygwin", MAX_PATH);
+  strncat (path, "/Cygwin", MAX_PATH - strlen(path) - 1);
   LogBabblePrintf ("Program directory for program link: %s", path);
   make_link (path, title, target, arg, iconpath);
 }
diff --git a/ini.h b/ini.h
index 4b9ed69..41ba8ec 100644
--- a/ini.h
+++ b/ini.h
@@ -22,7 +22,7 @@ class io_stream;
 
 typedef std::vector <std::string> IniList;
 extern IniList found_ini_list, setup_ext_list;
-const std::string setup_exts[] = { "xz", "bz2", "ini" };
+const std::string setup_exts[] = { "zst", "xz", "bz2", "ini" };
 extern bool is_64bit;
 extern bool is_new_install;
 extern std::string SetupArch;