3ddd76648930cee349a1d8f2b6dd467bd19ca7af
[feisty_meow.git] / nucleus / library / filesystem / heavy_file_ops.cpp
1 /*****************************************************************************\
2 *                                                                             *
3 *  Name   : heavy file operations                                             *
4 *  Author : Chris Koeritz                                                     *
5 *                                                                             *
6 *******************************************************************************
7 * Copyright (c) 2005-$now By Author.  This program is free software; you can  *
8 * redistribute it and/or modify it under the terms of the GNU General Public  *
9 * License as published by the Free Software Foundation; either version 2 of   *
10 * the License or (at your option) any later version.  This is online at:      *
11 *     http://www.fsf.org/copyleft/gpl.html                                    *
12 * Please send any updates to: fred@gruntose.com                               *
13 \*****************************************************************************/
14
15 #include "directory.h"
16 #include "filename.h"
17 #include "filename_list.h"
18 #include "heavy_file_ops.h"
19 #include "huge_file.h"
20
21 #include <basis/functions.h>
22 #include <basis/guards.h>
23 #include <structures/object_packers.h>
24
25 using namespace basis;
26 using namespace structures;
27
28 namespace filesystem {
29
30 #define DEBUG_HEAVY_FILE_OPS
31   // uncomment for noisier debugging.
32
33 #undef LOG
34 #ifdef DEBUG_HEAVY_FILE_OPS
35   #include <stdio.h>
36   #define LOG(to_print) printf("%s::%s: %s\n", static_class_name(), func, astring(to_print).s())
37 #else
38   #define LOG(s) {if(!!s){}}
39 #endif
40
41 //////////////
42
43 file_transfer_header::file_transfer_header(const file_time &time_stamp)
44 : _filename(),
45   _byte_start(0),
46   _length(0),
47   _time(time_stamp)
48 {
49 }
50
51 astring file_transfer_header::text_form() const
52 {
53   astring time_text;
54   _time.text_form(time_text);
55   return astring("file=") + _filename
56       + a_sprintf(" start=%d len=%d stamp=", _byte_start, _length)
57       + time_text;
58 }
59
60 astring file_transfer_header::readable_text_form() const
61 {
62   astring time_text;
63   _time.readable_text_form(time_text);
64   return _filename
65       + a_sprintf(" [%d bytes, mod ", _length)
66       + time_text + "]";
67 }
68
69 void file_transfer_header::pack(byte_array &packed_form) const
70 {
71   _filename.pack(packed_form);
72   attach(packed_form, _byte_start);
73   attach(packed_form, _length);
74   _time.pack(packed_form);
75 }
76
77 bool file_transfer_header::unpack(byte_array &packed_form)
78 {
79   if (!_filename.unpack(packed_form)) return false;
80   if (!detach(packed_form, _byte_start)) return false;
81   if (!detach(packed_form, _length)) return false;
82   if (!_time.unpack(packed_form)) return false;
83   return true;
84 }
85
86 int file_transfer_header::packed_size() const
87 {
88 byte_array temp;
89 attach(temp, _byte_start);
90 //hmmm: really ugly above; we should get a more exact way to know the size of
91 //      packed doubles.
92   return _filename.length() + 1
93       + temp.length()
94       + sizeof(int)
95       + _time.packed_size();
96 }
97
98 //////////////
99
100 const size_t heavy_file_operations::COPY_CHUNK_FACTOR = 1 * MEGABYTE;
101
102 size_t heavy_file_operations::copy_chunk_factor()
103 { return COPY_CHUNK_FACTOR; }
104
105 heavy_file_operations::~heavy_file_operations() {}
106   // we only need this due to our use of the root_object class_name support.
107
108 const char *heavy_file_operations::outcome_name(const outcome &to_name)
109 {
110   switch (to_name.value()) {
111     case SOURCE_MISSING: return "SOURCE_MISSING";
112     case TARGET_ACCESS_ERROR: return "TARGET_ACCESS_ERROR";
113     case TARGET_DIR_ERROR: return "TARGET_DIR_ERROR";
114     default: return common::outcome_name(to_name);
115   }
116 }
117
118 outcome heavy_file_operations::copy_file(const astring &source,
119     const astring &destination, int copy_chunk_factor)
120 {
121 #ifdef DEBUG_HEAVY_FILE_OPS
122   FUNCDEF("copy_file");
123 #endif
124   // check that the source exists...
125   filename source_path(source);
126   if (!source_path.exists()) return SOURCE_MISSING;
127   file_time source_time(source_path);  // get the time on the source.
128
129   // make sure the target directory exists...
130   filename target_path(destination);
131   filename targ_dir = target_path.dirname();
132   if (!directory::recursive_create(targ_dir.raw())) return TARGET_DIR_ERROR;
133
134   // open the source for reading.
135   huge_file source_file(source, "rb");
136   if (!source_file.good()) return SOURCE_MISSING;
137 //hmmm: could be source is not accessible instead.
138
139   // open target file for writing.
140   huge_file target_file(destination, "wb");
141   if (!target_file.good()) return TARGET_ACCESS_ERROR;
142
143   byte_array chunk;
144   int bytes_read = 0;
145   outcome ret;
146   while ( (ret = source_file.read(chunk, copy_chunk_factor, bytes_read))
147       == huge_file::OKAY) {
148     int bytes_stored;
149     ret = target_file.write(chunk, bytes_stored);
150     if (bytes_stored != bytes_read) return TARGET_ACCESS_ERROR;
151     if (source_file.eof()) break;  // time to escape.
152   }
153
154   // set the time on the target file from the source's time.
155   source_time.set_time(target_path);
156
157 #ifdef DEBUG_HEAVY_FILE_OPS
158   astring time;
159   source_time.text_form(time);
160   LOG(astring("setting file time for ") + source + " to " + time);
161 #endif
162
163   return OKAY;
164 }
165
166 outcome heavy_file_operations::write_file_chunk(const astring &target,
167     double byte_start, const byte_array &chunk, bool truncate,
168     int copy_chunk_factor)
169 {
170 #ifdef DEBUG_HEAVY_FILE_OPS
171 //  FUNCDEF("write_file_chunk");
172 #endif
173   if (byte_start < 0) return BAD_INPUT;
174
175   filename targ_name(target);
176   if (!directory::recursive_create(targ_name.dirname().raw()))
177     return TARGET_DIR_ERROR;
178
179   if (!targ_name.exists()) {
180     huge_file target_file(target, "w");
181   }
182
183   huge_file target_file(target, "r+b");
184     // open the file for updating (either read or write).
185   if (!target_file.good()) return TARGET_ACCESS_ERROR;
186   double curr_len = target_file.length();
187
188   if (curr_len < byte_start) {
189     byte_array new_chunk;
190     while (curr_len < byte_start) {
191       target_file.seek(0, byte_filer::FROM_END);  // go to the end of the file.
192       new_chunk.reset(minimum(copy_chunk_factor,
193           int(curr_len - byte_start + 1)));
194       int written;
195       outcome ret = target_file.write(new_chunk, written);
196       if (written < new_chunk.length()) return TARGET_ACCESS_ERROR;
197       curr_len = target_file.length();
198     }
199   }
200   target_file.seek(byte_start, byte_filer::FROM_START);
201     // jump to the proper location in the file.
202   int wrote;
203   outcome ret = target_file.write(chunk, wrote);
204   if (wrote != chunk.length()) return TARGET_ACCESS_ERROR;
205   if (truncate) {
206     target_file.truncate();
207   }
208   return OKAY;
209 }
210
211 basis::outcome heavy_file_operations::advance(const filename_list &to_transfer,
212     file_transfer_header &last_action)
213 {
214   FUNCDEF("advance");
215   int indy = to_transfer.locate(last_action._filename);
216   if (negative(indy)) return BAD_INPUT;  // error, file not found in list.
217   if (indy >= to_transfer.elements() - 1) return FINISHED;  // done.
218   const file_info *currfile = to_transfer.get(indy + 1);
219   last_action._filename = currfile->raw();
220   last_action._time = currfile->_time;
221
222 #ifdef DEBUG_HEAVY_FILE_OPS
223   if (currfile->_time == file_time(time_t(0)))
224     LOG(astring("failed for ") + currfile->raw() + " -- has zero file time");
225 #endif
226
227   last_action._byte_start = 0;
228   last_action._length = 0;
229   return OKAY;
230 }
231
232 outcome heavy_file_operations::buffer_files(const astring &source_root,
233     const filename_list &to_transfer, file_transfer_header &last_action,
234     byte_array &storage, int maximum_bytes)
235 {
236   FUNCDEF("buffer_files");
237   storage.reset();  // clear out the current contents.
238
239   if (!to_transfer.elements()) {
240     // we seem to be done.
241     return FINISHED;
242   }
243
244   outcome to_return = OKAY;
245
246   // start filling the array with bytes from the files.
247   while (storage.length() < maximum_bytes) {
248     double remaining_in_array = maximum_bytes - storage.length()
249         - last_action.packed_size();
250     if (remaining_in_array < 128) {
251       // ensure that we at least have a reasonable amount of space left
252       // for storing into the array.
253       break;
254     }
255
256     // find the current file we're at, as provided in record.
257     if (!last_action._filename) {
258       // no filename yet.  assume this is the first thing we've done.
259       const file_info *currfile = to_transfer.get(0);
260       last_action._filename = currfile->raw();
261       last_action._time = currfile->_time;
262       last_action._byte_start = 0;
263       last_action._length = 0;
264     }
265
266     const file_info *found = to_transfer.find(last_action._filename);
267     if (!found) {
268       // they have referenced a file that we don't have.  that's bad news.
269       return BAD_INPUT;
270     }
271
272     astring full_file = source_root + "/" + last_action._filename;
273     huge_file current(full_file, "rb");
274     if (!current.good()) {
275       // we need to skip this file.
276 LOG(astring("skipping bad file: ") + full_file);
277       to_return = advance(to_transfer, last_action);
278       if (to_return != OKAY) break;
279       continue;
280     }
281
282     if (last_action._byte_start + last_action._length >= current.length()) {
283 LOG(astring("finished stuffing file: ") + full_file);
284       // this file is done now.  go to the next one.
285       to_return = advance(to_transfer, last_action);
286       if (to_return != OKAY) break;
287       continue;
288     }
289
290     // calculate the largest piece remaining of that file that will fit in the
291     // allotted space.
292     double new_start = last_action._byte_start + last_action._length;
293     double remaining_in_file = current.length() - new_start;
294     if (remaining_in_file < 0) remaining_in_file = 0;
295     double new_len = minimum(remaining_in_file, remaining_in_array);
296     
297     // pack this new piece of the file.
298     current.seek(new_start, byte_filer::FROM_START);
299     byte_array new_chunk;
300     int bytes_read = 0;
301     outcome ret = current.read(new_chunk, int(new_len), bytes_read);
302     if (bytes_read != new_len) {
303       if (!bytes_read) {
304         // some kind of problem reading the file.
305         to_return = advance(to_transfer, last_action);
306         if (to_return != OKAY) break;
307         continue;
308       }
309 //why would this happen?  just complain, i guess.
310     }
311
312     // update the record since it seems we're successful here.
313     last_action._byte_start = new_start;
314     last_action._length = int(new_len);
315
316     // add in this next new chunk of file.
317     last_action.pack(storage);  // add the header.
318     storage += new_chunk;  // add the new stuff.
319
320     if (!current.length()) {
321       // ensure we don't get stuck redoing zero length files, which we allowed
322       // to go past their end above (since otherwise we'd never see them).
323       to_return = advance(to_transfer, last_action);
324       if (to_return != OKAY) break;
325       continue;
326     }
327     
328     // just keep going, if there's space...
329   }
330
331   return to_return;
332 }
333
334 } //namespace.
335