From 831c862a7fcd20137cb4002c6ea062011522f907 Mon Sep 17 00:00:00 2001 From: Achim Gratz Date: Wed, 1 Jul 2015 22:05:10 +0200 Subject: [PATCH 3/5] Refactor setup search and implement XZ compressed setup files * ini.cc: Construct setup_ext_list from array until we can use C++11 aggregate initializers. (decompress_ini): Refactored for use from do_local_ini and do_remote_ini. Change outdated comment about setup.ini uncompressed size. (check_ini_sig): Factor out signature check. (fetch_remote_ini): Refactored for use from do_remote_ini. (do_local_ini): Iterate over search results in found_ini_list. Use ini_decompress and check_ini_sig. (do_remote_ini): Iterate over known setup file extensions from setup_ext_list with early-out semantics, preferring ".xz" over ".bz2" over ".ini" extension. Use fetch_remote_ini and check_ini_sig. * ini.h: Remove unused macros. * IniParseFindVisitor.cc: Remove, the search is already done by SetupFindVisitor in do_from_local_dir. * IniParseFindVisitor.cc: Ditto. * Makefile.am (@SETUP@_SOURCES): Ditto. --- ChangeLog | 21 ++++ IniParseFindVisitor.cc | 116 ------------------ IniParseFindVisitor.h | 50 -------- Makefile.am | 2 - ini.cc | 319 +++++++++++++++++++++++++++++-------------------- ini.h | 4 - 6 files changed, 209 insertions(+), 303 deletions(-) delete mode 100644 IniParseFindVisitor.cc delete mode 100644 IniParseFindVisitor.h diff --git a/ChangeLog b/ChangeLog index e21e2e6..0d12e9b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,26 @@ 2015-07-01 Achim Gratz + * ini.cc: Construct setup_ext_list from array until we can use + C++11 aggregate initializers. + (decompress_ini): Refactored for use from do_local_ini and + do_remote_ini. Change outdated comment about setup.ini + uncompressed size. + (check_ini_sig): Factor out signature check. + (fetch_remote_ini): Refactored for use from do_remote_ini. + (do_local_ini): Iterate over search results in found_ini_list. + Use ini_decompress and check_ini_sig. + (do_remote_ini): Iterate over known setup file extensions from + setup_ext_list with early-out semantics, preferring ".xz" over + ".bz2" over ".ini" extension. Use fetch_remote_ini and + check_ini_sig. + * ini.h: Remove unused macros. + * IniParseFindVisitor.cc: Remove, the search is already done by + SetupFindVisitor in do_from_local_dir. + * IniParseFindVisitor.cc: Ditto. + * Makefile.am (@SETUP@_SOURCES): Ditto. + +2015-07-01 Achim Gratz + * fromcwd.cc: Remove unused includes. Add global found_ini_list to record the search result. (SetupFindVisitor): Make setup.{ini,bz2,xz} known and provide bool diff --git a/IniParseFindVisitor.cc b/IniParseFindVisitor.cc deleted file mode 100644 index 871d6de..0000000 --- a/IniParseFindVisitor.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2002,2007 Robert Collins. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * A copy of the GNU General Public License can be found at - * http://www.gnu.org/ - * - * Written by Robert Collins - * - */ - -#if 0 -static const char *cvsid = - "\n%%% $Id$\n"; -#endif - -#include "IniParseFindVisitor.h" - -#include "csu_util/rfc1738.h" - -#include "IniParseFeedback.h" -#include "IniDBBuilder.h" -#include "io_stream.h" -#include "ini.h" -#include - -using namespace std; - -extern int yyparse (); - -IniParseFindVisitor::IniParseFindVisitor(IniDBBuilder &aBuilder, - const std::string& localroot, - IniParseFeedback &feedback) - : _Builder (aBuilder), _feedback (feedback), baseLength (localroot.size()), - local_ini(0), setup_timestamp (0), setup_version() -{} - -IniParseFindVisitor::~IniParseFindVisitor(){} - -/* look for potential packages we can add to the in-memory package - * database - */ -void -IniParseFindVisitor::visitFile(const std::string& basePath, - const WIN32_FIND_DATA *theFile) -{ - //TODO: Test for case sensitivity issues - if (casecompare(SETUP_INI_FILENAME, theFile->cFileName)) - return; - - const char *dir = basePath.c_str () + basePath.size() - strlen (SETUP_INI_DIR); - if (dir < basePath.c_str ()) - return; - if ((dir != basePath.c_str () && dir[-1] != '/' && dir[-1] != '\\') || casecompare (SETUP_INI_DIR, dir)) - return; - - current_ini_name = basePath + theFile->cFileName; - - io_stream *ini_file = io_stream::open("file://" + current_ini_name, "rb", 0); - - if (!ini_file) - // We don't throw an exception, because while this is fatal to parsing, it - // isn't to the visitation. - { - // This should never happen - // If we want to handle it happening, use the log strategy call - throw new runtime_error ("IniParseFindVisitor: failed to open ini file, which should never happen"); - return; - } - - _feedback.babble("Found ini file - " + current_ini_name); - _feedback.iniName (current_ini_name); - - /* Copy leading part of path to temporary buffer and unescape it */ - - size_t pos = baseLength + 1; - size_t len = basePath.size () - (pos + strlen (SETUP_INI_DIR) + 1); - _Builder.parse_mirror = len <= 0 ? "" - : rfc1738_unescape (basePath.substr (pos, len)); - ini_init (ini_file, &_Builder, _feedback); - - /*yydebug = 1; */ - - if (yyparse () || yyerror_count > 0) - _feedback.error (yyerror_messages); - else - local_ini++; - - if (_Builder.timestamp > setup_timestamp) - { - setup_timestamp = _Builder.timestamp; - setup_version = _Builder.version; - } -} - -int -IniParseFindVisitor::iniCount() const -{ - return local_ini; -} - -unsigned int -IniParseFindVisitor::timeStamp () const -{ - return setup_timestamp; -} - -std::string -IniParseFindVisitor::version() const -{ - return setup_version; -} diff --git a/IniParseFindVisitor.h b/IniParseFindVisitor.h deleted file mode 100644 index ff48f70..0000000 --- a/IniParseFindVisitor.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2002,2007 Robert Collins. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * A copy of the GNU General Public License can be found at - * http://www.gnu.org/ - * - * Written by Robert Collins - * - */ - -#ifndef SETUP_INIPARSEFINDVISITOR_H -#define SETUP_INIPARSEFINDVISITOR_H - -#include "FindVisitor.h" - - -/* parse passed in setup.ini files from disk. */ -class IniDBBuilder; -class IniParseFeedback; -/* IniParse files and create a package db when no cached .ini exists */ -class IniParseFindVisitor : public FindVisitor -{ -public: - IniParseFindVisitor (IniDBBuilder &aBuilder, - const std::string& localroot, - IniParseFeedback &); - virtual void visitFile(const std::string& basePath, const WIN32_FIND_DATA *); - virtual ~ IniParseFindVisitor (); - - unsigned int timeStamp() const; - std::string version() const; - int iniCount() const; -protected: - IniParseFindVisitor (IniParseFindVisitor const &); - IniParseFindVisitor & operator= (IniParseFindVisitor const &); -private: - IniDBBuilder &_Builder; - IniParseFeedback &_feedback; - unsigned int baseLength; - int local_ini; - unsigned int setup_timestamp; - std::string setup_version; -}; - -#endif /* SETUP_INIPARSEFINDVISITOR_H */ diff --git a/Makefile.am b/Makefile.am index 88330d5..e9263cc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -163,8 +163,6 @@ inilint_SOURCES = \ iniparse.yy \ IniParseFeedback.cc \ IniParseFeedback.h \ - IniParseFindVisitor.cc \ - IniParseFindVisitor.h \ install.cc \ io_stream.cc \ io_stream.h \ diff --git a/ini.cc b/ini.cc index e90f98c..92642b3 100644 --- a/ini.cc +++ b/ini.cc @@ -39,7 +39,6 @@ #include "mount.h" #include "site.h" #include "find.h" -#include "IniParseFindVisitor.h" #include "IniParseFeedback.h" #include "io_stream.h" @@ -57,11 +56,12 @@ extern ThreeBarProgressPage Progress; unsigned int setup_timestamp = 0; std::string ini_setup_version; -std::string current_ini_sig_name; +IniList setup_ext_list( setup_exts, setup_exts+3); // TODO: use C++11x initializer lists static BoolOption NoVerifyOption (false, 'X', "no-verify", "Don't verify setup.ini signatures"); extern int yyparse (); + /*extern int yydebug;*/ class GuiParseFeedback : public IniParseFeedback @@ -119,16 +119,145 @@ private: unsigned int lastpct; }; +static io_stream* +decompress_ini (io_stream *ini_file) +{ + // Replace the current compressed setup stream with its decompressed + // version. Which decompressor to use is determined by file magic. + io_stream *compressed_stream = compress::decompress (ini_file); + if (!compressed_stream) + { + /* This isn't a valid compressed file. */ + // TODO: if we could determine that the original input stream + // was an uncompressed file to begin with we could always try to + // uncompress and skip the corresponding check. + delete compressed_stream; + delete ini_file; + ini_file = NULL; + } + else + { + /* Decompress the entire file in memory. This has the advantage + that input_stream->get_size() will work during parsing and + we'll have an accurate status bar. Also, we can't seek + compressed streams, so when we write out a local cached copy + of the .ini file below, we'd otherwise have to delete this + stream and uncompress it again from the start, which is + wasteful. The current uncompressed size of the setup.ini + file as of 2015 is about 5 MiB, so this is not a great deal + of memory. */ + io_stream *uncompressed = new io_stream_memory (); + /* Note that the decompress io_stream now "owns" the underlying + compressed io_stream instance, so it need not be deleted + explicitly. */ + if ((io_stream::copy (compressed_stream, uncompressed) != 0) || + (compressed_stream->error() != 0)) + { + /* There was a problem decompressing compressed_stream. */ + Log (LOG_PLAIN) << + "Warning: Error code " << compressed_stream->error() << + " occurred while uncompressing " << current_ini_name << + " - possibly truncated or corrupt file. " + " Retrying with uncompressed version." << endLog; + delete uncompressed; + ini_file = NULL; + } + else + { + ini_file = uncompressed; + ini_file->seek (0, IO_SEEK_SET); + } + } + return ini_file; +} + +static io_stream* +check_ini_sig (io_stream* ini_file, io_stream* ini_sig_file, + bool& sig_fail, const char* site, const char* sig_name, HWND owner) +{ + // Unless the NoVerifyOption is set, check the signature for the + // current setup and record the result. On a failed signature check + // the streams are invalidated so even if we tried to read in the + // setup anyway there's be nothing to parse. + if (!NoVerifyOption && ini_file) + { + if (!ini_sig_file) + { + note (owner, IDS_SETUPINI_MISSING, sig_name, site); + delete ini_file; + ini_file = NULL; + sig_fail = true; + } + else if (!verify_ini_file_sig (ini_file, ini_sig_file, owner)) + { + note (owner, IDS_SIG_INVALID, sig_name, site); + delete ini_sig_file; + ini_sig_file = NULL; + delete ini_file; + ini_file = NULL; + sig_fail = true; + } + } + return ini_file; +} + static int -do_local_ini (HWND) +do_local_ini (HWND owner) { + size_t ini_count = 0; GuiParseFeedback myFeedback; - IniDBBuilderPackage findBuilder(myFeedback); - IniParseFindVisitor myVisitor (findBuilder, local_dir, myFeedback); - Find (local_dir).accept(myVisitor, 2); // Only search two levels deep. - setup_timestamp = myVisitor.timeStamp(); - ini_setup_version = myVisitor.version(); - return myVisitor.iniCount(); + IniDBBuilderPackage aBuilder(myFeedback); + io_stream *ini_file, *ini_sig_file; + // iterate over all setup files found in do_from_local_dir + for (IniList::const_iterator n = found_ini_list.begin(); + n != found_ini_list.end(); ++n) + { + bool sig_fail = false; + std::string current_ini_ext, current_ini_name, current_ini_sig_name; + + current_ini_name = *n; + current_ini_sig_name = current_ini_name + ".sig"; + current_ini_ext = current_ini_name.substr(current_ini_name.rfind(".") + 1); + ini_sig_file = io_stream::open("file://" + current_ini_sig_name, "rb", 0); + ini_file = io_stream::open("file://" + current_ini_name, "rb", 0); + ini_file = check_ini_sig (ini_file, ini_sig_file, sig_fail, + "localdir", current_ini_sig_name.c_str(), owner); + // did we find a compressed setup? + if (ini_file && + !(casecompare (current_ini_ext, "xz") && + casecompare (current_ini_ext, "bz2"))) + ini_file = decompress_ini(ini_file); + if (!ini_file || sig_fail) + { + // no setup found or signature invalid + note (owner, IDS_SETUPINI_MISSING, SetupBaseName.c_str(), + "localdir"); + } + else + { + // grok information from setup + myFeedback.babble("Found ini file - " + current_ini_name); + myFeedback.iniName (current_ini_name); + aBuilder.parse_mirror = ""; + ini_init (ini_file, &aBuilder, myFeedback); + + /*yydebug = 1; */ + + if (yyparse () || yyerror_count > 0) + myFeedback.error (yyerror_messages); + else + ++ini_count; + + if (aBuilder.timestamp > setup_timestamp) + { + setup_timestamp = aBuilder.timestamp; + ini_setup_version = aBuilder.version; + } + delete ini_file; + ini_file = NULL; + } + } + return ini_count; } static int @@ -140,146 +269,75 @@ do_remote_ini (HWND owner) io_stream *ini_file, *ini_sig_file; /* FIXME: Get rid of this io_stream pointer travesty. The need to - explicitly delete these things is ridiculous. Note that the - decompress io_stream "owns" the underlying compressed io_stream - instance, so it should not be deleted explicitly. */ + explicitly delete these things is ridiculous. */ + // iterate over all sites for (SiteList::const_iterator n = site_list.begin(); n != site_list.end(); ++n) { bool sig_fail = false; - /* First try to fetch the .bz2 compressed ini file. */ - current_ini_name = n->url + SETUP_INI_DIR + SETUP_BZ2_FILENAME; - current_ini_sig_name = n->url + SETUP_INI_DIR + SETUP_BZ2_FILENAME + ".sig"; - ini_file = get_url_to_membuf (current_ini_name, owner); - if (!NoVerifyOption) - ini_sig_file = get_url_to_membuf (current_ini_sig_name, owner); - if (!NoVerifyOption && ini_file && !ini_sig_file) + std::string current_ini_ext, current_ini_name, current_ini_sig_name; + // iterate over known extensions for setup + for (IniList::const_iterator ext = setup_ext_list.begin(); + ext != setup_ext_list.end(); + ext++) { - note (owner, IDS_SETUPINI_MISSING, current_ini_sig_name.c_str(), n->url.c_str()); - delete ini_file; - ini_file = NULL; - sig_fail = true; + current_ini_ext = *ext; + current_ini_name = n->url + SetupIniDir + SetupBaseName + "." + current_ini_ext; + current_ini_sig_name = current_ini_name + ".sig"; + ini_sig_file = get_url_to_membuf (current_ini_sig_name, owner); + ini_file = get_url_to_membuf (current_ini_name, owner); + ini_file = check_ini_sig (ini_file, ini_sig_file, sig_fail, + n->url.c_str(), current_ini_sig_name.c_str(), owner); + // stop searching as soon as we find a setup file + if (ini_file) + break; } - else if (!NoVerifyOption && ini_file && !verify_ini_file_sig (ini_file, ini_sig_file, owner)) + // did we find a compressed setup? + if (ini_file && + !(casecompare (current_ini_ext, "xz") && + casecompare (current_ini_ext, "bz2"))) + ini_file = decompress_ini(ini_file); + if (!ini_file || sig_fail) { - note (owner, IDS_SIG_INVALID, current_ini_sig_name.c_str(), n->url.c_str()); - delete ini_file; - ini_file = NULL; - delete ini_sig_file; - ini_sig_file = NULL; - sig_fail = true; + // no setup found or signature invalid + note (owner, IDS_SETUPINI_MISSING, SetupBaseName.c_str(), n->url.c_str()); } - if (ini_file) + else { - /* Decompress the entire file in memory right now. This has the - advantage that input_stream->get_size() will work during parsing - and we'll have an accurate status bar. Also, we can't seek - bz2 streams, so when it comes time to write out a local cached - copy of the .ini file below, we'd otherwise have to delete this - stream and uncompress it again from the start, which is wasteful. - The current uncompresed size of the .ini file as of 2007 is less - than 600 kB, so this is not a great deal of memory. */ - io_stream *bz2_stream = compress::decompress (ini_file); - if (!bz2_stream) - { - /* This isn't a valid bz2 file. */ - delete ini_file; - ini_file = NULL; - } + // grok information from setup + myFeedback.iniName (current_ini_name); + aBuilder.parse_mirror = n->url; + ini_init (ini_file, &aBuilder, myFeedback); + + /*yydebug = 1; */ + + if (yyparse () || yyerror_count > 0) + myFeedback.error (yyerror_messages); else { - io_stream *uncompressed = new io_stream_memory (); - - if ((io_stream::copy (bz2_stream, uncompressed) != 0) || - (bz2_stream->error() != 0)) - { - /* There was a problem decompressing bz2. */ - ini_file = NULL; - Log (LOG_PLAIN) << - "Warning: Error code " << bz2_stream->error() << " occurred while uncompressing " << - current_ini_name << " - possibly truncated or corrupt bzip2" - " file. Retrying with uncompressed version." << endLog; - delete bz2_stream; - delete uncompressed; - } - else + /* save known-good setup.ini locally */ + const std::string fp = "file://" + local_dir + "/" + + rfc1738_escape_part (n->url) + + "/" + SetupIniDir + SetupBaseName + ".ini"; + io_stream::mkpath_p (PATH_TO_FILE, fp, 0); + if (io_stream *out = io_stream::open (fp, "wb", 0)) { - delete bz2_stream; - ini_file = uncompressed; ini_file->seek (0, IO_SEEK_SET); + if (io_stream::copy (ini_file, out) != 0) + io_stream::remove (fp); + delete out; } + ++ini_count; } - } - - if (!ini_file) - { - /* Try to look for a plain .ini file because one of the following - happened above: - - there was no .bz2 file found on the mirror. - - the .bz2 file didn't look like a valid bzip2 file. - - there was an error during bzip2 decompression. */ - current_ini_name = n->url + SETUP_INI_DIR + SETUP_INI_FILENAME; - current_ini_sig_name = n->url + SETUP_INI_DIR + SETUP_INI_FILENAME + ".sig"; - ini_file = get_url_to_membuf (current_ini_name, owner); - if (!NoVerifyOption) - ini_sig_file = get_url_to_membuf (current_ini_sig_name, owner); - - if (!NoVerifyOption && ini_file && !ini_sig_file) - { - note (owner, IDS_SETUPINI_MISSING, current_ini_sig_name.c_str(), n->url.c_str()); - delete ini_file; - ini_file = NULL; - sig_fail = true; - } - else if (!NoVerifyOption && ini_file && !verify_ini_file_sig (ini_file, ini_sig_file, owner)) - { - note (owner, IDS_SIG_INVALID, current_ini_sig_name.c_str(), n->url.c_str()); - delete ini_file; - ini_file = NULL; - delete ini_sig_file; - ini_sig_file = NULL; - sig_fail = true; - } - } - - if (!ini_file) - { - if (!sig_fail) - note (owner, IDS_SETUPINI_MISSING, SETUP_INI_FILENAME, n->url.c_str()); - continue; - } - - myFeedback.iniName (current_ini_name); - aBuilder.parse_mirror = n->url; - ini_init (ini_file, &aBuilder, myFeedback); - - /*yydebug = 1; */ - - if (yyparse () || yyerror_count > 0) - myFeedback.error (yyerror_messages); - else - { - /* save known-good setup.ini locally */ - const std::string fp = "file://" + local_dir + "/" + - rfc1738_escape_part (n->url) + - "/" + SETUP_INI_DIR + SETUP_INI_FILENAME; - io_stream::mkpath_p (PATH_TO_FILE, fp, 0); - if (io_stream *out = io_stream::open (fp, "wb", 0)) + if (aBuilder.timestamp > setup_timestamp) { - ini_file->seek (0, IO_SEEK_SET); - if (io_stream::copy (ini_file, out) != 0) - io_stream::remove (fp); - delete out; + setup_timestamp = aBuilder.timestamp; + ini_setup_version = aBuilder.version; } - ++ini_count; - } - if (aBuilder.timestamp > setup_timestamp) - { - setup_timestamp = aBuilder.timestamp; - ini_setup_version = aBuilder.version; + delete ini_file; + ini_file = NULL; } - delete ini_file; } return ini_count; } @@ -374,4 +432,3 @@ do_ini (HINSTANCE h, HWND owner) DWORD threadID; CreateThread (NULL, 0, do_ini_thread_reflector, context, 0, &threadID); } - diff --git a/ini.h b/ini.h index c2903ef..164e3d2 100644 --- a/ini.h +++ b/ini.h @@ -48,10 +48,6 @@ typedef enum EXCLUDE_NOT_FOUND } excludes; -#define SETUP_INI_DIR SetupIniDir.c_str() -#define SETUP_INI_FILENAME (SetupBaseName+".ini").c_str() -#define SETUP_BZ2_FILENAME (SetupBaseName+".bz2").c_str() - /* The following three vars are used to facilitate error handling between the parser/lexer and its callers, namely ini.cc:do_remote_ini() and IniParseFindVisitor::visitFile(). */ -- 2.4.3