feisty meow concerns codebase  2.140
bundle_creator.cpp
Go to the documentation of this file.
1 
2 //hmmm: anything related to _stub_size should be kept, but that is where
3 // we need a redundant search mechanism that can't be fooled so easily
4 // by modifying exe; make a pattern that will be found and is the first
5 // place to start looking for manifest.
6 
7 /*****************************************************************************\
8 * *
9 * Name : bundle_creator *
10 * Author : Chris Koeritz *
11 * *
12 *******************************************************************************
13 * Copyright (c) 2006-$now By Author. This program is free software; you can *
14 * redistribute it and/or modify it under the terms of the GNU General Public *
15 * License as published by the Free Software Foundation; either version 2 of *
16 * the License or (at your option) any later version. This is online at: *
17 * http://www.fsf.org/copyleft/gpl.html *
18 * Please send any updates to: fred@gruntose.com *
19 \*****************************************************************************/
20 
21 #include "common_bundle.h"
22 
25 #include <basis/array.h>
26 #include <basis/byte_array.h>
27 #include <basis/environment.h>
31 #include <filesystem/byte_filer.h>
32 #include <filesystem/directory.h>
33 #include <filesystem/filename.h>
34 #include <filesystem/file_time.h>
35 #include <loggers/console_logger.h>
36 #include <loggers/file_logger.h>
40 #include <textual/byte_formatter.h>
41 #include <textual/list_parsing.h>
42 #include <textual/parser_bits.h>
43 #include <timely/time_stamp.h>
44 
45 #include <stdio.h>
46 #include <sys/stat.h>
47 #include <zlib.h>
48 //#ifdef __WIN32__
49  //#include <io.h>
50 //#endif
51 
52 using namespace application;
53 using namespace basis;
54 using namespace configuration;
55 using namespace filesystem;
56 using namespace loggers;
57 using namespace filesystem;
58 using namespace processes;
59 using namespace structures;
60 using namespace textual;
61 using namespace timely;
62 
63 const int CHUNKING_SIZE = 256 * KILOBYTE;
64  // we'll read this big a chunk from a source file at a time.
65 
66 const astring SUBVERSION_FOLDER = ".svn";
67  // we don't want to include this in a bundle.
68 
69 #define BASE_LOG(to_print) program_wide_logger::get().log(to_print, ALWAYS_PRINT)
70 #define LOG(to_print) CLASS_EMERGENCY_LOG(program_wide_logger::get(), to_print)
71 
72 //#define DEBUG_BUNDLER
73  // uncomment for noisy debugging version.
74 
75 // returns the "retval" and mentions that this is a failure at "where".
76 #define FAIL_RETURN(retval, where) { \
77  LOG(astring("failure in ") + where + a_sprintf(", exit=%d", retval)); \
78  return retval; \
79 }
80 
82 
83 bool true_value(const astring &value)
84 { return (!value.equal_to("0")) && (!value.equal_to("false")); }
85 
87 
88 // this structure overrides the manifest_chunk by providing a source string.
89 
90 struct bundled_chunk : manifest_chunk
91 {
92  astring _source;
93  virtual ~bundled_chunk() {}
94 };
95 
97 
98 // main bundler class.
99 
100 class bundle_creator : public application_shell
101 {
102 public:
103  bundle_creator()
104  : application_shell(),
105  _app_name(filename(_global_argv[0]).basename()),
106  _bundle(NULL_POINTER), _stub_size(0), _keyword() {}
107 
108  virtual ~bundle_creator() {
109  WHACK(_bundle);
110  }
111 
112  DEFINE_CLASS_NAME("bundle_creator");
113  virtual int execute();
114  int print_instructions();
115 
116  astring determine_stub_file_and_validate();
118 
119  int open_output_file();
121 
123  int read_manifest();
125 
127  int write_stub_and_toc();
129 
130  int bundle_sources();
132 
133  int finalize_file();
135 
136  int write_offset();
138 
140  int patch_recursive_target(const astring &source, const astring &target,
141  int manifest_index);
143 
147  int recurse_into_dir(const astring &source, const astring &target,
148  int manifest_index);
150 
151  int patch_wildcard_target(const astring &source, const astring &target,
152  int manifest_index);
154 
156  int add_files_here(directory &dirndl, const astring &source,
157  const astring &target, int manifest_index);
159 
160  bool get_file_size(const astring &file, un_int &size, byte_array &timestamp);
162 
163 private:
164  astring _app_name;
165  astring _output_file;
166  astring _manifest_file;
167  array<bundled_chunk> _manifest_list;
168  byte_filer *_bundle;
169  int _stub_size;
170  astring _keyword; // set if we were given a keyword on cmd line.
171 };
172 
174 
176 {
178 %s: This program needs two parameters on the command line.\n\
179 The -o flag must point at the bundled output file to create. The -m flag\n\
180 must point at a valid manifest file that defines what will be packed into\n\
181 the output file. See the example manifest in the bundler example\n\
182 (in setup_src/bundle_example) for more information on the required file\n\
183 format.\n\
184 ", _app_name.s()));
185  return 4;
186 }
187 
188 int bundle_creator::execute()
189 {
190  FUNCDEF("execute");
191 
192  BASE_LOG(astring("starting file bundling at ") + time_stamp::notarize(false));
193 
195 
196 //BASE_LOG(astring("before starting, cmds has: ") + parser_bits::platform_eol_to_chars() + cmds.text_form());
197 
198  astring temp;
199  if (cmds.get_value('?', temp)) return print_instructions();
200  if (cmds.get_value("?", temp)) return print_instructions();
201  if (!cmds.get_value('o', _output_file)) return print_instructions();
202  if (!cmds.get_value('m', _manifest_file)) return print_instructions();
203 
204  if (filename(_output_file).exists()) {
206 %s: The output file already exists. Please move it out of\n\
207 the way; this program will not overwrite existing files.\n",
208 _app_name.s()));
209  return 3;
210  }
211 
212  if (!filename(_manifest_file).exists()) {
214 %s: The manifest file does not exist. This program cannot do anything\n\
215 without a valid packing manifest.\n", _app_name.s()));
216  return 2;
217  }
218 
219  // test this early on so we don't waste time uselessly.
220  astring stub_file_okay = determine_stub_file_and_validate();
221  if (!stub_file_okay) {
223 %s: The unpacking stub file does not exist (check binaries folder).\n\
224 Abandoning bundling process.\n", _app_name.s()));
225  return 4;
226  }
227 
228  // make sure we snag any keyword that was passed on the command line.
229  cmds.get_value("keyword", _keyword);
230 
231  // first step is to provide some built-in variables that can be used to
232  // make the manifests less platform specific. this doesn't really help
233  // if you bundle it on linux and try to run it on windows. but either
234  // platform's resources can easily be made into a bundle with the same
235  // packing manifest.
236 #ifndef __WIN32__
237  environment::set("EXE_END", ""); // executable file ending.
238  environment::set("DLL_START", "lib"); // dll file prefix.
239  environment::set("DLL_END", ".so"); // dll file ending.
240 #else
241  environment::set("EXE_END", ".exe");
242  environment::set("DLL_START", "");
243  environment::set("DLL_END", ".dll");
244 #endif
245  // specify a target variable on the source side so that we can operate in there,
246  // even if the bundle doesn't specify one. otherwise we can't run source side commands
247  // properly if the paths are based on TARGET (like TMP often is).
248  environment::set("TARGET", environment::TMP());
249 
250  int ret = 0;
251  if ( (ret = read_manifest()) ) FAIL_RETURN(ret, "reading manifest");
252  // read manifest to build list of what's what.
253  if ( (ret = open_output_file()) ) FAIL_RETURN(ret, "opening output file");
254  // open up our output file for the bundled chunks.
255  if ( (ret = write_stub_and_toc()) ) FAIL_RETURN(ret, "writing stub and TOC");
256  // writes the stub unpacker application and the table of contents to the
257  // output file.
258  if ( (ret = bundle_sources()) ) FAIL_RETURN(ret, "bundling source files");
259  // stuff all the source files into the output bundle.
260  if ( (ret = finalize_file()) ) FAIL_RETURN(ret, "finalizing file");
261  // finishes with the file and closes it up.
262  if ( (ret = write_offset()) ) FAIL_RETURN(ret, "writing offset");
263  // stores the offset of the TOC into the output file in a special location
264  // that is delineated by a known keyword (muftiloc) and which should only
265  // exist in the file in one location.
266 
267  return 0;
268 }
269 
270 int bundle_creator::open_output_file()
271 {
272  FUNCDEF("open_output_file");
273  _bundle = new byte_filer(_output_file, "wb");
274  if (!_bundle->good()) {
275  LOG(astring("failed to open the output file: ") + _output_file);
276  return 65;
277  }
278  return 0;
279 }
280 
281 bool bundle_creator::get_file_size(const astring &infile, un_int &size,
283 {
284  FUNCDEF("get_file_size");
285  time_stamp.reset();
286  // access the source file to get its size.
287  byte_filer source_file(infile, "rb");
288  if (!source_file.good()) {
289  LOG(astring("could not access the file for size check: ") + infile);
290  return false;
291  }
292  size = int(source_file.length());
293  file_time tim(infile);
294  tim.pack(time_stamp);
295  return true;
296 }
297 
298 int bundle_creator::add_files_here(directory &dirndl, const astring &source,
299  const astring &target, int manifest_index)
300 {
301  FUNCDEF("add_files_here");
302  for (int i = 0; i < dirndl.files().length(); i++) {
303  astring curry = dirndl.files()[i];
304  // skip .svn folders and contents.
305  if (curry.contains(SUBVERSION_FOLDER)) continue;
306 //hmmm: this could be a much nicer generalized file exclusion list.
307 
308 //LOG(astring("file is: ") + curry);
309  bundled_chunk new_guy;
310  new_guy._source = source + "/" + curry; // the original full path to it.
311  new_guy._payload = target + "/" + curry;
312  new_guy._keywords = _manifest_list[manifest_index]._keywords;
313  // copy the flags from the parent, so we don't forget options.
314  new_guy._flags = _manifest_list[manifest_index]._flags;
315  // remove some flags that make no sense for the new guy.
316  new_guy._flags &= ~RECURSIVE_SRC;
317 
318 //LOG(a_sprintf("adding: source=%s targ=%s", new_guy._source.s(), new_guy._payload.s()));
319  bool okaysize = get_file_size(new_guy._source, new_guy._size, new_guy.c_filetime);
320  if (!okaysize || (new_guy._size < 0) ) {
321  LOG(astring("failed to get file size for ") + new_guy._source);
322  return 75;
323  }
324 
325  _manifest_list.insert(manifest_index + 1, 1);
326  _manifest_list[manifest_index + 1] = new_guy;
327  }
328  return 0;
329 }
330 
331 int bundle_creator::recurse_into_dir(const astring &source,
332  const astring &target, int manifest_index)
333 {
334  FUNCDEF("recurse_into_dir");
335 //LOG(astring("src=") + source + " dest=" + target);
336 
337  // we won't include the subversion folder.
338  if (source.contains(SUBVERSION_FOLDER)) return 0;
339 
340  string_array dirs; // culled from the directory listing.
341  {
342  // don't pay for the directory object on the recursive invocation stack;
343  // just have what we need on the stack (the directory list).
344  directory dirndl(source);
345 //check dir for goodness!
346  int ret = add_files_here(dirndl, source, target, manifest_index);
347  // add in just the files that were found.
348  if (ret != 0) {
349  // this is a failure, but the function complains about it already.
350  return 75;
351  }
352  dirs = dirndl.directories();
353  }
354 
355 //LOG("now scanning directories...");
356 
357  // now scan across the directories we found.
358  for (int i = 0; i < dirs.length(); i++) {
359  astring s = dirs[i];
360 //LOG(astring("curr dir is ") + s);
361  int ret = recurse_into_dir(source + "/" + s, target + "/"
362  + s, manifest_index);
363  if (ret != 0) return ret; // bail out.
364  }
365 
366  return 0;
367 }
368 
369 int bundle_creator::patch_recursive_target(const astring &source,
370  const astring &target, int manifest_index)
371 {
372  FUNCDEF("patch_recursive_target");
373 //LOG(astring("patch recurs src=") + source + " targ=" + target);
374  return recurse_into_dir(source, target, manifest_index);
375 }
376 
377 int bundle_creator::patch_wildcard_target(const astring &source,
378  const astring &target, int manifest_index)
379 {
380  FUNCDEF("patch_wildcard_target");
381  // find the last slash. the rest is our wildcard component.
382  int src_end = source.end();
383  int slash_indy = source.find('/', src_end, true);
384  astring real_source = source.substring(0, slash_indy - 1);
385  astring wild_pat = source.substring(slash_indy + 1, src_end);
386 //BASE_LOG(astring("got src=") + real_source + " wildpat=" + wild_pat);
387 
388  directory dirndl(real_source, wild_pat.s());
389 //check dir for goodness!
390  int ret = add_files_here(dirndl, real_source, target, manifest_index);
391  if (ret != 0) {
392  // this is a failure, but the function complains about it already.
393  return 75;
394  }
395 
396  return 0;
397 }
398 
399 int bundle_creator::read_manifest()
400 {
401  FUNCDEF("read_manifest");
402  ini_configurator ini(_manifest_file, configurator::RETURN_ONLY);
403  string_table toc;
404  bool worked = ini.get_section("toc", toc);
405  if (!worked) {
406  LOG(astring("failed to read TOC section in manifest:\n") + _manifest_file
407  + "\ndoes that file exist?");
408  return 65;
409  }
410 
411 //hmmm: make a class member.
412  file_logger noisy_logfile(application_configuration::make_logfile_name
413  ("bundle_creator_activity.log"));
414  noisy_logfile.log(astring('-', 76));
415  noisy_logfile.log(astring("Bundling starts at ") + time_stamp::notarize(false));
416 
417  // add enough items in the list for our number of sections.
418  _manifest_list.insert(0, toc.symbols());
419  astring value; // temporary string used below.
420  int final_return = 0; // if non-zero, an error occurred.
421 
422 #define BAIL(retval) \
423  final_return = retval; \
424  toc.zap_index(i); \
425  _manifest_list.zap(i, i); \
426  i--; \
427  continue
428 
429  for (int i = 0; i < toc.symbols(); i++) {
430  // read all the info in this section and store it into our list.
431  astring section_name = toc.name(i);
432  section_name.strip_spaces(astring::FROM_FRONT);
433  if (section_name[0] == '#') {
434 //hmmm: this looks a bit familiar from bail macro above. abstract out?
435  toc.zap_index(i);
436  _manifest_list.zap(i, i);
437  i--;
438  continue; // skip comments.
439  }
440 
441  // check for any keywords on the section. these are still needed for
442  // variables, which otherwise would skip the rest of the field checks.
443  if (ini.get(section_name, "keyword", value)) {
445  string_array keys;
446  bool worked = list_parsing::parse_csv_line(value, keys);
447  if (!worked) {
448  LOG(astring("failed to parse keywords for section ")
449  + section_name + " in " + _manifest_file);
450  BAIL(82);
451  }
453  _manifest_list[i]._keywords = keys;
454  astring dumped;
455  list_parsing::create_csv_line(_manifest_list[i]._keywords, dumped);
456  noisy_logfile.log(section_name + " keywords: " + dumped);
457  }
458 
459  if (ini.get(section_name, "variable", value)) {
460  // this is a variable assignment. it is the only thing we care about
461  // for this section, so the rest is ignored.
462  variable_tokenizer zohre;
463  zohre.parse(value);
464  if (zohre.symbols() < 1) {
465  LOG(astring("failed to parse a variable statement from ") + value);
466  BAIL(37);
467  }
468  _manifest_list[i]._flags = SET_VARIABLE; // not orred, just this.
469  // set the two parts of our variable.
470  _manifest_list[i]._payload = zohre.table().name(0);
471  _manifest_list[i]._parms = zohre.table()[0];
472  BASE_LOG(astring("will set ") + _manifest_list[i]._payload + " = "
473  + _manifest_list[i]._parms);
474  astring new_value = parser_bits::substitute_env_vars(_manifest_list[i]._parms);
475  environment::set(_manifest_list[i]._payload, new_value);
476 
477 #ifdef DEBUG_BUNDLER
478  BASE_LOG(astring("** variable ") + _manifest_list[i]._payload + " should have value=" + new_value);
479  BASE_LOG(astring("** variable ") + _manifest_list[i]._payload + " now does have value=" + environment::get(_manifest_list[i]._payload));
480 #endif
481 
482  continue;
483  } else if (ini.get(section_name, "assert_defined", value)) {
484  // they are just asking for a variable test, to see if a variable
485  // that the installer needs is actually defined at unpacking time.
486  _manifest_list[i]._payload = value;
487  _manifest_list[i]._flags = TEST_VARIABLE_DEFINED;
488  BASE_LOG(astring("will test ") + _manifest_list[i]._payload + " is "
489  + "defined at unpacking time.");
490  continue;
491  }
492 
493  if (!ini.get(section_name, "source", _manifest_list[i]._source)) {
494  // check whether they told us not to pack and it's executable.
495  bool okay_to_omit_source = false;
496  astring value2;
497  if (ini.get(section_name, "no_pack", value)
498  && ini.get(section_name, "exec_target", value2) ) {
499  if (true_value(value) && true_value(value2)) {
500  // this type of section doesn't need source declared.
501  okay_to_omit_source = true;
502  }
503  }
504  if (!okay_to_omit_source) {
505  LOG(astring("failed to read the source entry for section ")
506  + section_name + " in " + _manifest_file);
507  BAIL(67);
508  }
509  }
510  // fix meshugener backslashes so we can count on the slash direction.
511  _manifest_list[i]._source.replace_all('\\', '/');
512 
513  if (!ini.get(section_name, "target", _manifest_list[i]._payload)) {
514  // check whether they told us not to pack and it's executable.
515  bool okay_to_omit_target = false;
516  astring value2;
517  if (ini.get(section_name, "no_pack", value)
518  && ini.get(section_name, "exec_source", value2) ) {
519  if (true_value(value) && true_value(value2)) {
520  // this type of section doesn't need target declared.
521  okay_to_omit_target = true;
522  }
523  }
524  if (!okay_to_omit_target) {
525  LOG(astring("failed to read the target entry for section ")
526  + section_name + " in " + _manifest_file);
527  BAIL(68);
528  }
529  }
530  // fix backslashes in target also.
531  _manifest_list[i]._payload.replace_all('\\', '/');
532 
533  // capture any parameters they have specified for exec or other options.
534  if (ini.get(section_name, "parms", value)) {
535  _manifest_list[i]._parms = value;
536 #ifdef DEBUG_BUNDLER
537  BASE_LOG(astring("got parms for ") + section_name + " as: " + value);
538 #endif
539  if (value[0] != '"') {
540  // repair the string if we're running on windows.
541  _manifest_list[i]._parms = astring("\"") + value + "\"";
542  }
543  noisy_logfile.log(section_name + " parms: " + _manifest_list[i]._parms);
544  }
545 
546  // check for the ignore errors flag.
547  if (ini.get(section_name, "error_okay", value)) {
548  if (true_value(value))
549  _manifest_list[i]._flags |= IGNORE_ERRORS;
550  }
551 
552  // see if they are saying not to overwrite the target file.
553  if (ini.get(section_name, "no_replace", value)) {
554  if (true_value(value))
555  _manifest_list[i]._flags |= NO_OVERWRITE;
556  }
557 
558  // test whether they are saying not to complain about a failure with
559  // our normal pop-up dialog (on winders).
560  if (ini.get(section_name, "quiet", value)) {
561  if (true_value(value))
562  _manifest_list[i]._flags |= QUIET_FAILURE;
563  }
564 
565  // did they want a backup of the original to be made, instead of
566  // just overwriting the file?
567  if (ini.get(section_name, "make_backup", value)) {
568  if (true_value(value))
569  _manifest_list[i]._flags |= MAKE_BACKUP_FILE;
570  }
571 
572  // look for our recursion flag.
573  if (ini.get(section_name, "recurse", value)) {
574  if (true_value(value))
575  _manifest_list[i]._flags |= RECURSIVE_SRC;
576  } else {
577  // the options here are only appropriate when the target is NOT set to
578  // be recursive.
579 
580  if (ini.get(section_name, "no_pack", value)) {
581  // allow either side to not be required if this is an executable.
582  if (true_value(value))
583  _manifest_list[i]._flags |= OMIT_PACKING;
584  }
585 
586  // check if they have specified a source side executable.
587  if (ini.get(section_name, "exec_source", value)) {
588  if (true_value(value)) {
589  _manifest_list[i]._flags |= SOURCE_EXECUTE;
590  }
591  } else {
592  // check if they have specified a target side executable. this is
593  // mutually exclusive with a source side exec.
594  if (ini.get(section_name, "exec_target", value)) {
595  if (true_value(value))
596  _manifest_list[i]._flags |= TARGET_EXECUTE;
597  }
598  }
599  }
600 
601  // replace environment variables in the source now...
602  _manifest_list[i]._source = parser_bits::substitute_env_vars
603  (_manifest_list[i]._source, false);
604 
605  // look for wildcards in the source.
606  int indy = _manifest_list[i]._source.find("*");
607 
608  // see if they specified a keyword on the command line and if this matches.
609  // if not we need to abandon this item.
610  if (!!_keyword && !_manifest_list[i]._keywords.member(_keyword)) {
611  // their keyword choice didn't match what we were told to use.
612  noisy_logfile.log(astring("skipping ") + _manifest_list[i]._payload
613  + " file check; doesn't match keyword \"" + _keyword + "\"");
614  continue;
615  }
616 
617  // we only access the source file here if it's finalized. we can't do
618  // this if the target is supposed to be recursive or if it's got a wildcard
619  // pattern in it.
620  if (!(_manifest_list[i]._flags & RECURSIVE_SRC) && negative(indy)
621  && !(_manifest_list[i]._flags & OMIT_PACKING) ) {
622  // access the source file to get its size.
623  byte_filer source_file(_manifest_list[i]._source, "rb");
624  if (!source_file.good()) {
625  LOG(astring("could not access the source file for bundling: ")
626  + _manifest_list[i]._source);
627  BAIL(69);
628  }
629  bool okaysize = get_file_size(_manifest_list[i]._source,
630  _manifest_list[i]._size, _manifest_list[i].c_filetime);
631  if (!okaysize || (_manifest_list[i]._size < 0) ) {
632  // this is a failure, but the function complains about it already.
633  BAIL(75);
634  }
635  }
636  }
637 
638  // patch the manifest list for wildcards and recursive sources.
639  for (int i = 0; i < _manifest_list.length(); i++) {
640  bundled_chunk curr = _manifest_list[i];
641 
642  if (!!_keyword && !curr._keywords.member(_keyword)) {
643  // this item's keyword doesn't match the one we were given, so skip it.
644  noisy_logfile.log(astring("zapping entry for ") + curr._payload
645  + "; doesn't match keyword \"" + _keyword + "\"");
646  _manifest_list.zap(i, i);
647  i--; // skip back since we eliminated an index.
648  continue;
649  }
650 
651  if (curr._flags & SET_VARIABLE) {
652  // we're done working on this.
653  continue;
654  } else if (curr._flags & TEST_VARIABLE_DEFINED) {
655  // this also requires no further effort.
656  continue;
657  } else if (curr._flags & RECURSIVE_SRC) {
658  // handle a recursive style target.
659  int star_indy = curr._source.find("*");
660  if (non_negative(star_indy)) {
661  // this is currently illegal. we don't allow recursion + wildcards.
662  LOG(astring("illegal combination of recursion and wildcard: ")
663  + curr._source);
664  BAIL(70);
665  }
666  // handle the recursive guy.
667  int ret = patch_recursive_target(curr._source, curr._payload, i);
668  if (ret != 0) {
669  LOG(astring("failed during packing of recursive source: ")
670  + curr._source);
671  BAIL(72);
672  }
673  // take this item out of the picture, since all contents got included.
674  _manifest_list.zap(i, i);
675  i--; // skip back since we eliminated an index.
676  continue;
677  } else if (curr._flags & SOURCE_EXECUTE) {
678  // we have massaged the current manifest chunk as much as we can, so now
679  // we will execute the source item if that was specified.
680  BASE_LOG(astring("launching ") + curr._source);
681  if (!!curr._parms) {
682  curr._parms = parser_bits::substitute_env_vars(curr._parms, false);
683  BASE_LOG(astring("\tparameters ") + curr._parms);
684  }
685  BASE_LOG(astring('-', 76));
686  basis::un_int kid;
687  basis::un_int retval = launch_process::run(curr._source, curr._parms,
688  launch_process::AWAIT_APP_EXIT, kid);
689  if (retval != 0) {
690  LOG(astring("failed to launch process, source=") + curr._source
691  + ", with parms " + curr._parms);
692  if (! (curr._flags & IGNORE_ERRORS) ) {
693  BAIL(92);
694  }
695  }
696  BASE_LOG(astring('-', 76));
697  if (curr._flags & OMIT_PACKING) {
698  // this one shouldn't be included in the package.
699  _manifest_list.zap(i, i);
700  i--; // skip back since we eliminated an index.
701  }
702  continue;
703  } else {
704  // check for a wildcard.
705  int star_indy = curr._source.find("*");
706  if (negative(star_indy)) continue; // simple targets are boring.
707  // this does have a wildcard in it. let's make sure it's in the right
708  // place for a wildcard in our scheme.
709  int slash_indy = curr._source.find('/', curr._source.end(), true);
710  if (star_indy < slash_indy) {
711  BASE_LOG(astring("illegal wildcard placement in ") + curr._source);
712  BASE_LOG(astring(" (the wildcard must be in the last component of the path)"));
713  BAIL(71);
714  }
715  // handle the wildcarded source.
716  int ret = patch_wildcard_target(curr._source, curr._payload, i);
717  if (ret != 0) {
718  LOG(astring("failed during packing of wildcarded source: ")
719  + curr._source);
720  BAIL(73);
721  }
722  _manifest_list.zap(i, i);
723  i--; // skip back since we eliminated an index.
724  continue;
725  }
726  }
727 
728 #ifdef DEBUG_BUNDLER
729  if (!final_return) {
730  // we had a successful run so we can print this stuff out.
731  LOG("read the following info from manifest:");
732  for (int i = 0; i < _manifest_list.length(); i++) {
733  bundled_chunk &curr = _manifest_list[i];
734  BASE_LOG(a_sprintf("(%d) size %d, %s => %s", i, curr._size,
735  curr._source.s(), curr._payload.s()));
736  }
737  }
738 #endif
739 
740  return final_return;
741 }
742 
743 astring bundle_creator::determine_stub_file_and_validate()
744 {
745  FUNCDEF("determine_stub_file_and_validate");
746  // define our location to find the unpacking stub program.
747 //hmmm: make this a command line parameter.
748 #ifdef __UNIX__
749  astring stub_filename("unpacker_stub");
750 #endif
751 #ifdef __WIN32__
752  astring stub_filename("unpacker_stub.exe");
753 #endif
754  astring repo_dir = "$RUNTIME_PATH";
755  astring stub_file = parser_bits::substitute_env_vars
756  (repo_dir + "/binaries/" + stub_filename, false);
757  if (!filename(stub_file).exists()) {
758  // we needed to find that to build the bundle.
759  LOG(astring("could not find unpacking stub file at: ") + stub_file);
760  return astring::empty_string();
761  }
762  return stub_file;
763 }
764 
765 int bundle_creator::write_stub_and_toc()
766 {
767  FUNCDEF("write_stub_and_toc");
768 
769  astring stub_file = determine_stub_file_and_validate();
770  if (!stub_file) return 1;
771 
772  // make sure the stub is accessible.
773  byte_filer stubby(stub_file, "rb");
774  if (!stubby.good()) {
775  FAIL_RETURN(80, astring("could not read the unpacking stub at: ") + stub_file);
776  }
777  _stub_size = int(stubby.length()); // get the stub size for later reference.
778  byte_array whole_stub;
779  stubby.read(whole_stub, _stub_size + 100);
780  stubby.close();
781  _bundle->write(whole_stub);
782 
783  byte_array packed_toc_len;
784  structures::obscure_attach(packed_toc_len, _manifest_list.length());
785  int ret = _bundle->write(packed_toc_len);
786  if (ret < 0) {
787  LOG(astring("could not write the TOC length to the bundle: ")
788  + _output_file);
789  return 81;
790  }
791 
792  // dump out the manifest list in our defined format.
793  for (int i = 0; i < _manifest_list.length(); i++) {
794  bundled_chunk &curr = _manifest_list[i];
795 //LOG(a_sprintf("flag %d is %d", i, curr._flags));
796  byte_array chunk;
797  curr.pack(chunk);
798  if (_bundle->write(chunk) <= 0) {
799  LOG(a_sprintf("could not write item #%d [%s] to the bundle: ", i,
800  curr._source.s())
801  + _output_file);
802  return 88;
803  }
804  }
805 
806  return 0;
807 }
808 
809 int bundle_creator::bundle_sources()
810 {
811  FUNCDEF("bundle_sources");
812  // go through all the source files and append them to the bundled output.
813  file_logger noisy_logfile(application_configuration::make_logfile_name
814  ("bundle_creator_activity.log"));
815  for (int i = 0; i < _manifest_list.length(); i++) {
816  bundled_chunk &curr = _manifest_list[i];
817 
818  if (curr._flags & SET_VARIABLE) {
819  // all we need to do is keep this in the manifest.
820  noisy_logfile.log(astring("bundling: variable setting ") + curr._payload
821  + "=" + curr._parms);
822  continue;
823  } else if (curr._flags & TEST_VARIABLE_DEFINED) {
824  // just remember to test this when running the unpack.
825  noisy_logfile.log(astring("bundling: test variable ") + curr._payload
826  + " is defined.");
827  continue;
828  } else if (curr._flags & OMIT_PACKING) {
829  // this one shouldn't be included in the package.
830  continue;
831  }
832 
833  noisy_logfile.log(astring("bundling: ") + curr._source);
834  byte_filer source(curr._source, "rb");
835  if (!source.good()) {
836  LOG(a_sprintf("could not read item #%d for the bundle: \"", i)
837  + curr._source + "\"");
838  return 98;
839  }
840 
841  byte_array compressed(256 * KILOBYTE); // expand the buffer to start with.
842  byte_array temp; // temporary read buffer.
843 
844  // chew on the file a chunk at a time. this allows us to easily handle
845  // arbitrarily large files rather than reading their entirety into memory.
846  int total_written = 0;
847  do {
848  int ret = source.read(temp, CHUNKING_SIZE);
849  if (ret < 0) {
850  LOG(a_sprintf("failed while reading item #%d: ", i) + curr._source);
851  return 99;
852  }
853  total_written += ret; // add in what we expect to write.
854  // skip compressing if there's no data.
855  uLongf destlen = 0;
856  bool null_chunk = false;
857  if (ret == 0) {
858  compressed.reset();
859  null_chunk = true;
860  } else {
861  compressed.reset(int(0.1 * ret) + ret + KILOBYTE);
862  // provide some extra space as per zlib instructions. we're giving it
863  // way more than they request.
864  destlen = compressed.length();
865  // pack the chunks first so we can know sizes needed.
866  int comp_ret = compress(compressed.access(), &destlen, temp.observe(),
867  temp.length());
868  if (comp_ret != Z_OK) {
869  LOG(a_sprintf("failed while compressing item #%d: ", i)
870  + curr._source);
871  return 99;
872  }
873  compressed.zap(destlen, compressed.length() - 1);
874  }
875  byte_array just_sizes;
876  structures::obscure_attach(just_sizes, temp.length());
877  // add in the real size.
878  structures::obscure_attach(just_sizes, int(destlen));
879  // add in the packed size.
880  ret = _bundle->write(just_sizes);
881  if (ret <= 0) {
882  LOG(a_sprintf("failed while writing sizes for item #%d: ", i)
883  + curr._source);
884  return 93;
885  }
886  if (!null_chunk) {
887  ret = _bundle->write(compressed);
888  if (ret <= 0) {
889  LOG(a_sprintf("failed while writing item #%d: ", i) + curr._source);
890  return 93;
891  } else if (ret != compressed.length()) {
892  LOG(a_sprintf("wrote different size for item #%d (tried %d, "
893  "wrote %d): ", i, compressed.length(), ret) + curr._source);
894  return 93;
895  }
896  }
897  } while (!source.eof());
898 //hmmm: very common code to above size writing.
899  byte_array just_sizes;
900  structures::obscure_attach(just_sizes, -1);
901  structures::obscure_attach(just_sizes, -1);
902  int ret = _bundle->write(just_sizes);
903  if (ret <= 0) {
904  LOG(a_sprintf("failed while writing sentinel of item #%d: ", i)
905  + curr._source);
906  return 96;
907  }
908  source.close();
909  if (total_written != curr._size) {
910  LOG(a_sprintf("size (%d) disagrees with initial size (%d) for "
911  "item #%d: ", total_written, curr._size, i) + curr._source);
912  }
913  }
914  noisy_logfile.log(astring("Bundling run ends at ") + time_stamp::notarize(false));
915  noisy_logfile.log(astring('-', 76));
916 
917  return 0;
918 }
919 
920 int bundle_creator::finalize_file()
921 {
922  _bundle->close();
923  return 0;
924 }
925 
926 int bundle_creator::write_offset()
927 {
928  FUNCDEF("write_offset");
929  byte_filer bun(_output_file, "r+b"); // open the file for updating.
930 
931  astring magic_string("muftiloc"); // our sentinel string.
932  astring temp_string; // data from the file.
933 
934  while (!bun.eof()) {
935  // find the telltale text in the file.
936  bool found_it = false; // we'll set this to true if we see the string.
937  int location = 0; // where the sentinel's end is.
938  for (int i = 0; i < magic_string.length(); i++) {
939  int ret = bun.read(temp_string, 1);
940  if (ret <= 0) break;
941  if (temp_string[0] != magic_string[i]) break; // no match.
942  if (i == magic_string.end()) {
943  // we found a match to our string!
944  found_it = true;
945  location = int(bun.tell());
946 //LOG(a_sprintf("found the sentinel in the file! posn=%d", location));
947  }
948  }
949  if (!found_it) continue; // keep reading.
950  bun.seek(location);
951  byte_array packed_offset;
952  structures::obscure_attach(packed_offset, _stub_size);
953 //LOG(astring("pattern of len is:\n") + byte_format::text_dump(packed_offset));
954  // write the offset into the current position, which should be just after
955  // the sentinel's location.
956  bun.write(packed_offset);
957 //LOG(a_sprintf("wrote manifest offset before posn=%d", bun.tell()));
958  break; // done with looking for that pattern.
959  }
960  bun.close(); // completely finished now.
961 
962  chmod(_output_file.s(), 0766);
963  // make sure it's an executable file when we're done with it.
964 
965  BASE_LOG(astring("done file bundling at ") + time_stamp::notarize(false));
966 
967  return 0;
968 }
969 
971 
972 HOOPLE_MAIN(bundle_creator, )
973 
974 #ifdef __BUILD_STATIC_APPLICATION__
975  // static dependencies found by buildor_gen_deps.sh:
978  #include <basis/astring.cpp>
979  #include <basis/common_outcomes.cpp>
980  #include <basis/environment.cpp>
981  #include <basis/mutex.cpp>
982  #include <basis/utf_conversion.cpp>
989  #include <filesystem/byte_filer.cpp>
990  #include <filesystem/directory.cpp>
991  #include <filesystem/filename.cpp>
992  #include <filesystem/file_time.cpp>
993  #include <loggers/combo_logger.cpp>
994  #include <loggers/console_logger.cpp>
995  #include <loggers/critical_events.cpp>
996  #include <loggers/file_logger.cpp>
999  #include <structures/bit_vector.cpp>
1000  #include <structures/checksums.cpp>
1003  #include <structures/string_hasher.cpp>
1004  #include <structures/string_table.cpp>
1006  #include <textual/byte_formatter.cpp>
1007  #include <textual/list_parsing.cpp>
1008  #include <textual/parser_bits.cpp>
1010  #include <timely/earth_time.cpp>
1011  #include <timely/time_control.cpp>
1012  #include <timely/time_stamp.cpp>
1013 #endif // __BUILD_STATIC_APPLICATION__
1014 
#define chmod
Definition: Xos2defs.h:12
const astring SUBVERSION_FOLDER
#define BASE_LOG(to_print)
bool true_value(const astring &value)
#define FAIL_RETURN(retval, where)
const int CHUNKING_SIZE
#define BAIL(retval)
#define LOG(to_print)
int print_instructions(bool good, const astring &program_name)
Definition: checker.cpp:45
The application_shell is a base object for console programs.
a_sprintf is a specialization of astring that provides printf style support.
Definition: astring.h:440
const contents * observe() const
Returns a pointer to the underlying C array of data.
Definition: array.h:172
int length() const
Returns the current reported length of the allocated C array.
Definition: array.h:115
Provides a dynamically resizable ASCII character string.
Definition: astring.h:35
const char * s() const
synonym for observe. the 's' stands for "string", if that helps.
Definition: astring.h:113
bool substring(astring &target, int start, int end) const
a version that stores the substring in an existing "target" string.
Definition: astring.cpp:865
void strip_spaces(how_to_strip way=FROM_BOTH_SIDES)
removes excess space characters from string's beginning, end or both.
Definition: astring.h:325
bool equal_to(const char *that) const
returns true if "that" is equal to this.
Definition: astring.cpp:159
int end() const
returns the index of the last (non-null) character in the string.
Definition: astring.h:86
int find(char to_find, int position=0, bool reverse=false) const
Locates "to_find" in "this".
Definition: astring.cpp:574
bool contains(const astring &to_find) const
Returns true if "to_find" is contained in this string or false if not.
Definition: astring.cpp:162
A very common template for a dynamic array of bytes.
Definition: byte_array.h:36
Supports a configurator-based interface on text initialization files.
Manages a bank of textual definitions of variables.
const structures::string_table & table() const
provides a constant peek at the string_table holding the values.
int symbols() const
returns the number of entries in the variable_tokenizer.
bool parse(const basis::astring &to_tokenize)
parses the string using our established sentinel characters.
Provides file managment services using the standard I/O support.
Definition: byte_filer.h:32
Implements a scanner that finds all filenames in the directory specified.
Definition: directory.h:27
const structures::string_array & directories() const
these are the directory names from the folder.
Definition: directory.cpp:145
const structures::string_array & files() const
returns the list of files that we found in this directory.
Definition: directory.cpp:143
Provides operations commonly needed on file names.
Definition: filename.h:64
An array of strings with some additional helpful methods.
Definition: string_array.h:32
Provides a symbol_table that holds strings as the content.
Definition: string_table.h:32
basis::outcome zap_index(int index)
zaps the entry at the specified index. slower than whack().
Definition: symbol_table.h:403
const basis::astring & name(int index) const
returns the name held at the "index".
Definition: symbol_table.h:272
int symbols() const
returns the number of symbols listed in the table.
Definition: symbol_table.h:241
Represents a point in time relative to the operating system startup time.
Definition: time_stamp.h:38
void reset()
sets the stamp time back to now.
Definition: time_stamp.cpp:59
@ TEST_VARIABLE_DEFINED
check for required variable's presence.
Definition: common_bundle.h:42
@ OMIT_PACKING
for a source side exe, do not pack the file.
Definition: common_bundle.h:36
@ SET_VARIABLE
this item just has a variable assignment.
Definition: common_bundle.h:37
@ RECURSIVE_SRC
source is a recursive folder.
Definition: common_bundle.h:35
@ SOURCE_EXECUTE
the file should be executed before bundling.
Definition: common_bundle.h:33
@ QUIET_FAILURE
when errors happen, no popup message happens.
Definition: common_bundle.h:40
@ IGNORE_ERRORS
if set, errors in an item will not stop program.
Definition: common_bundle.h:38
@ TARGET_EXECUTE
the file should be executed on unbundling.
Definition: common_bundle.h:34
@ MAKE_BACKUP_FILE
save a copy if original file already exists.
Definition: common_bundle.h:41
@ NO_OVERWRITE
target file will not be overwritten if exists.
Definition: common_bundle.h:39
#define NULL_POINTER
The value representing a pointer to nothing.
Definition: definitions.h:32
#define DEFINE_CLASS_NAME(objname)
Defines the name of a class by providing a couple standard methods.
Definition: enhance_cpp.h:45
#define FUNCDEF(func_in)
FUNCDEF sets the name of a function (and plugs it into the callstack).
Definition: enhance_cpp.h:57
Provides macros that implement the 'main' program of an application.
#define HOOPLE_MAIN(obj_name, obj_args)
options that should work for most unix and linux apps.
Definition: hoople_main.h:61
Implements an application lock to ensure only one is running at once.
char ** _global_argv
The guards collection helps in testing preconditions and reporting errors.
Definition: array.h:30
void WHACK(contents *&ptr)
deletion with clearing of the pointer.
Definition: functions.h:121
bool non_negative(const type &a)
non_negative returns true if "a" is greater than or equal to zero.
Definition: functions.h:45
unsigned int un_int
Abbreviated name for unsigned integers.
Definition: definitions.h:62
bool negative(const type &a)
negative returns true if "a" is less than zero.
Definition: functions.h:43
const int KILOBYTE
Number of bytes in a kilobyte.
Definition: definitions.h:134
A platform independent way to obtain the timestamp of a file.
Definition: byte_filer.cpp:37
A logger that sends to the console screen using the standard output device.
A dynamic container class that holds any kind of object via pointers.
Definition: amorph.h:55
void obscure_attach(byte_array &packed_form, un_int to_attach)
like the normal attach but shifts in some recognizable sentinel data.
#include <time.h>
Definition: earth_time.cpp:37
we will read the manifest pieces out of our own exe image.
Definition: common_bundle.h:54