From: Chris Koeritz Date: Tue, 22 Jan 2013 02:24:50 +0000 (-0500) Subject: changes that may have fixed the file synch bugs, finally. added a new X-Git-Tag: 2.140.90~1133 X-Git-Url: https://feistymeow.org/gitweb/?p=feisty_meow.git;a=commitdiff_plain;h=c93ddbe5db0e3295f90f12685f96d86a060f4cd0 changes that may have fixed the file synch bugs, finally. added a new stage before tree compare where we make the local directory hierarchy on the target. this seems to cure all the stupidity issues we were having before. well, the issues in regard to the behavior of file_synch at least. --- diff --git a/nucleus/library/filesystem/directory_tree.cpp b/nucleus/library/filesystem/directory_tree.cpp index 621b73cd..54a8ca9b 100644 --- a/nucleus/library/filesystem/directory_tree.cpp +++ b/nucleus/library/filesystem/directory_tree.cpp @@ -496,9 +496,9 @@ bool directory_tree::calculate(bool just_size) bool directory_tree::calculate(filename_tree *start, bool just_size) { FUNCDEF("calculate"); -//#ifdef DEBUG_DIRECTORY_TREE +#ifdef DEBUG_DIRECTORY_TREE LOG(astring("calculate: got tree to start with at ") + start->_dirname.raw()); -//#endif +#endif dir_tree_iterator *ted = start_at(start, directory_tree::postfix); // create our iterator to do a postfix traversal. why postfix? well, // prefix has been used elsewhere and since it doesn't really matter what @@ -993,6 +993,46 @@ outcome directory_tree::remove_path(const astring &zap_item) return common::OKAY; } -} //namespace. +basis::outcome directory_tree::make_directories(const basis::astring new_root) +{ + FUNCDEF("make_directories"); + // iterate down the tree, making all the directories requested. + dir_tree_iterator *ted = start(directory_tree::prefix); + + int depth; // current depth in tree. + filename curr; // the current path the iterator is at. + string_array files; // the filenames held at the iterator. + +#ifdef DEBUG_DIRECTORY_TREE + LOG(astring("new root is ") + new_root); +#endif + filename old_root = path(); +#ifdef DEBUG_DIRECTORY_TREE + LOG(astring("old root was ") + old_root.raw()); +#endif + while (current(*ted, curr, files)) { + // we have a good directory to show. + directory_tree::depth(*ted, depth); + int found = curr.find(old_root); + if (found < common::OKAY) { + LOG(astring("failed to find prefix of path '") + old_root + + "' in the current directory '" + curr + "'"); + return found; + } + curr = curr.substring(found + old_root.length(), curr.end()); +//LOG(astring("curr now is: ") + curr); + filename dir_to_make(new_root + "/" + curr); +#ifdef DEBUG_DIRECTORY_TREE + LOG(astring("creating dir: ") + dir_to_make.raw()); +#endif + directory::recursive_create(dir_to_make.raw()); + next(*ted); + } + + throw_out(ted); + return common::OKAY; +} + +} //namespace. diff --git a/nucleus/library/filesystem/directory_tree.h b/nucleus/library/filesystem/directory_tree.h index 9c89ba43..6ad56a0e 100644 --- a/nucleus/library/filesystem/directory_tree.h +++ b/nucleus/library/filesystem/directory_tree.h @@ -104,6 +104,9 @@ public: removed in the filesystem. if the item is still really there, then the next rescan will put it back into the tree. */ + basis::outcome make_directories(const basis::astring new_root); + //!< creates all of the directories in this object, but start at the "new_root". + static bool compare_trees(const directory_tree &source, const directory_tree &target, filename_list &differences, file_info::file_similarity how_to_compare); diff --git a/nucleus/library/tests_filesystem/test_directory_tree.cpp b/nucleus/library/tests_filesystem/test_directory_tree.cpp index 756138ca..f0a84671 100644 --- a/nucleus/library/tests_filesystem/test_directory_tree.cpp +++ b/nucleus/library/tests_filesystem/test_directory_tree.cpp @@ -190,6 +190,25 @@ LOG(diffs.text_form()); ASSERT_FALSE(diffs.elements(), "no differences for reverse compare identical dirs"); } + { + // sixth test: see if the make_directories function works. +LOG("reading tree to recreate"); + directory_tree dir(path, pattern.s()); + ASSERT_TRUE(dir.good(), "makedirs test directory reading"); + filename tmpdir(environment::get("TMP") + "/zz_balfazzaral"); + LOG(astring("will write to tmp in ") + tmpdir); + basis::outcome result = dir.make_directories(tmpdir.raw()); + ASSERT_EQUAL(result.value(), common::OKAY, "makedirs should succeed"); + + +LOG("what happened with that? did it work?"); + +//hmmm: compare the directories with what we expect to be made; +// do a dirtree iterator on the path, and make sure each of those exists in the target place. + + } + + // nth test: // combine the results of the second test with a comparison like in the // third test. delete all of those temporary files that were added. diff --git a/octopi/library/tentacles/file_transfer_infoton.h b/octopi/library/tentacles/file_transfer_infoton.h index 2ea10e2a..0d24c207 100644 --- a/octopi/library/tentacles/file_transfer_infoton.h +++ b/octopi/library/tentacles/file_transfer_infoton.h @@ -31,6 +31,10 @@ class file_transfer_infoton : public infoton public: //! the commands specify what this package is intended to do. enum commands { + BUILD_TARGET_TREE = 4, + //!< asks the target side to build the directory tree from the source. + /*!< this is a new first step for the transfer. we want to make sure + the target will look right for what we're going to transfer over. */ TREE_COMPARISON = 1, //!< the destination root will be compared with the source root. /*!< the packed data in the request holds a packed symbol tree @@ -38,7 +42,7 @@ public: is the packed filename list that represents the differences between the two hierarchies. this is considered to start a file transfer based on those differences. */ - PLACE_FILE_CHUNKS, + PLACE_FILE_CHUNKS = 2, //!< the destination side requests a new set of chunks. /*!< this is based on the source's memory of where the transfer is at. this will only perform properly when the file transfer was requested to @@ -47,7 +51,7 @@ public: number of pairs of @code [ file_transfer_header + file chunk described in header ] @endcode */ - CONCLUDE_TRANSFER_MARKER + CONCLUDE_TRANSFER_MARKER = 3, //!< this infoton marks the end of the transfer process. /*!< we've added this type of transfer infoton to handle the finish of the transfer. previously this was marked by a null data packet, diff --git a/octopi/library/tentacles/file_transfer_tentacle.cpp b/octopi/library/tentacles/file_transfer_tentacle.cpp index 50cc5114..60bb2d6f 100644 --- a/octopi/library/tentacles/file_transfer_tentacle.cpp +++ b/octopi/library/tentacles/file_transfer_tentacle.cpp @@ -47,7 +47,7 @@ const int FTT_CLEANING_INTERVAL = 30 * SECOND_ms; const int TRANSFER_TIMEOUT = 10 * MINUTE_ms; // if it hasn't been touched in this long, it's out of there. -#define DEBUG_FILE_TRANSFER_TENTACLE +//#define DEBUG_FILE_TRANSFER_TENTACLE // uncomment for noisier version. #undef LOG @@ -460,6 +460,100 @@ outcome file_transfer_tentacle::reconstitute(const string_array &classifier, // the "handle_" and "conclude_" methods are thread-safe because the mutex is locked before // their invocations. +basis::outcome file_transfer_tentacle::handle_build_target_tree_request(file_transfer_infoton &req, + const octopus_request_id &item_id) +{ + FUNCDEF("handle_build_target_tree_request"); + + // get the mapping from the specified location on this side. + filename splitting(req._src_root); + string_array pieces; + bool rooted; + splitting.separate(rooted, pieces); + astring source_mapping = pieces[0]; + + // patch the name up to find the sub_path for the source. + filename source_start; + pieces.zap(0, 0); + source_start.join(rooted, pieces); + + // locate the allowed transfer depot for the mapping they provided. + file_transfer_record *mapping_record = _correspondences->find_mapping(source_mapping); + if (!mapping_record) { + LOG(astring("could not find source mapping of ") + source_mapping); + return NOT_FOUND; + } + + // unpack the tree that they sent us which describes their local area. + directory_tree *dest_tree = new directory_tree; + if (!dest_tree->unpack(req._packed_data)) { + LOG(astring("could not unpack requester's directory tree")); + WHACK(dest_tree); + return GARBAGE; + } + + string_array requested_names; + if (!requested_names.unpack(req._packed_data)) { + LOG(astring("could not unpack requester's filename includes")); + WHACK(dest_tree); + return GARBAGE; + } + + // look up to see if this is about something that has already been seen. + // we don't want to add a new transfer record if they're already working on + // this. that also lets them do a new tree compare to restart the transfer. + file_transfer_record *the_rec = _transfers->find(item_id._entity, + req._src_root, req._dest_root); + if (!the_rec) { + // there was no existing record; we'll create a new one. + the_rec = new file_transfer_record; + the_rec->_ent = item_id._entity; + the_rec->_src_root = req._src_root; + the_rec->_dest_root = req._dest_root; + _transfers->append(the_rec); + } else { + // record some activity on this record. + the_rec->_done = false; + the_rec->_last_active.reset(); + } + + // create any directories that are missing at this point. + basis::outcome result = dest_tree->make_directories(req._dest_root); + if (result != common::OKAY) { + LOG("ERROR: got bad result from make_directories!"); + } + + req._packed_data.reset(); // clear out existing stuff before cloning. + file_transfer_infoton *reply = dynamic_cast(req.clone()); + + reply->_request = false; // it's a response now. + reply->_success = result; + store_product(reply, item_id); + // send back the comparison list. + + return OKAY; +} + +basis::outcome file_transfer_tentacle::handle_build_target_tree_response(file_transfer_infoton &resp, + const octopus_request_id &item_id) +{ +//go to next step, tree comparison. +//look at the handle tree compare response for help though. + FUNCDEF("handle_build_target_tree_response"); + file_transfer_record *the_rec = _transfers->find(item_id._entity, + resp._src_root, resp._dest_root); + if (!the_rec) { + LOG(astring("could not find the record for this transfer: item=") + + item_id.text_form() + " src=" + resp._src_root + " dest=" + + resp._dest_root); + return NOT_FOUND; // not registered, so reject it. + } + + the_rec->_last_active.reset(); // record some activity on this record. + + return resp._success; +} + outcome file_transfer_tentacle::handle_tree_compare_request (file_transfer_infoton &req, const octopus_request_id &item_id) { @@ -542,7 +636,6 @@ outcome file_transfer_tentacle::handle_tree_compare_request filename req_curr(requested_names[j]); if (req_curr.compare_suffix(diff_curr)) { found = true; -//LOG(astring("will use: ") + req_curr); break; } } @@ -559,13 +652,9 @@ outcome file_transfer_tentacle::handle_tree_compare_request // before the client starts the transfer. reply->_request = false; // it's a response now. -LOG("storing product from transfer processing"); store_product(reply, item_id); // send back the comparison list. -LOG("now showing bin before return:"); -LOG(get_storage()->text_form()); - return OKAY; } @@ -705,7 +794,7 @@ outcome file_transfer_tentacle::handle_storage_response astring full_file = resp._dest_root + filename::default_separator() + recorded_info->secondary(); -LOG(astring("telling it to write to fullfile: ") + full_file); +//LOG(astring("telling it to write to fullfile: ") + full_file); outcome ret = heavy_file_operations::write_file_chunk(full_file, found._byte_start, to_write); @@ -798,6 +887,10 @@ outcome file_transfer_tentacle::consume(infoton &to_chow, AUTO_LOCK; // protect our lists while we're working on them. switch (inf->_command) { + case file_transfer_infoton::BUILD_TARGET_TREE: { + if (inf->_request) return handle_build_target_tree_request(*inf, item_id); + else return handle_build_target_tree_response(*inf, item_id); + } case file_transfer_infoton::TREE_COMPARISON: { if (inf->_request) return handle_tree_compare_request(*inf, item_id); else return handle_tree_compare_response(*inf, item_id); diff --git a/octopi/library/tentacles/file_transfer_tentacle.h b/octopi/library/tentacles/file_transfer_tentacle.h index fc1edb48..bcb8d8d5 100644 --- a/octopi/library/tentacles/file_transfer_tentacle.h +++ b/octopi/library/tentacles/file_transfer_tentacle.h @@ -167,17 +167,21 @@ private: int _mode; //!< how will the comparison be done? // these process the request and response infotons that are passed to us. + basis::outcome handle_build_target_tree_request(file_transfer_infoton &req, + const octopus_request_id &item_id); + basis::outcome handle_build_target_tree_response(file_transfer_infoton &resp, + const octopus_request_id &item_id); basis::outcome handle_tree_compare_request(file_transfer_infoton &req, const octopus_request_id &item_id); - basis::outcome handle_tree_compare_response(file_transfer_infoton &req, + basis::outcome handle_tree_compare_response(file_transfer_infoton &resp, const octopus_request_id &item_id); basis::outcome handle_storage_request(file_transfer_infoton &req, const octopus_request_id &item_id); - basis::outcome handle_storage_response(file_transfer_infoton &req, + basis::outcome handle_storage_response(file_transfer_infoton &resp, const octopus_request_id &item_id); basis::outcome conclude_storage_request(file_transfer_infoton &req, const octopus_request_id &item_id); - basis::outcome conclude_storage_response(file_transfer_infoton &req, + basis::outcome conclude_storage_response(file_transfer_infoton &resp, const octopus_request_id &item_id); }; diff --git a/octopi/library/tentacles/recursive_file_copy.cpp b/octopi/library/tentacles/recursive_file_copy.cpp index 3272ad0f..b012ef08 100644 --- a/octopi/library/tentacles/recursive_file_copy.cpp +++ b/octopi/library/tentacles/recursive_file_copy.cpp @@ -85,7 +85,9 @@ outcome recursive_file_copy::copy_hierarchy(int transfer_mode, } */ - astring source_root = "snootums"; + const astring transfer_name = "snootums"; + + astring source_root = transfer_name; if (source_start.t()) { source_root += filename::default_separator() + source_start; } @@ -95,35 +97,73 @@ outcome recursive_file_copy::copy_hierarchy(int transfer_mode, (MAX_CHUNK_RFC_COPY_HIER, (file_transfer_tentacle::transfer_modes)transfer_mode); ring_leader.add_tentacle(tran); - outcome add_ret = tran->add_correspondence("snootums", source_dir, + outcome add_ret = tran->add_correspondence(transfer_name, source_dir, EXPECTED_MAXIMUM_TRANSFER_TIME); if (add_ret != tentacle::OKAY) RETURN_ERROR_RFC("failed to add the correspondence", NOT_FOUND); +//hmmm: this kind of object creation should be packaged in helper functions. file_transfer_infoton *initiate = new file_transfer_infoton; initiate->_request = true; - initiate->_command = file_transfer_infoton::TREE_COMPARISON; + initiate->_command = file_transfer_infoton::BUILD_TARGET_TREE; initiate->_src_root = source_root; initiate->_dest_root = target_dir; - directory_tree target_area(target_dir); -//hmmm: simple asset counting debugging in calculate would be nice too. - target_area.calculate( !(transfer_mode & file_transfer_tentacle::COMPARE_CONTENT_SAMPLE) ); - initiate->package_tree_info(target_area, includes); + + // make a directory snapshot with just directories, no files. + directory_tree target_area_just_dirs(source_dir, "*", true); + initiate->package_tree_info(target_area_just_dirs, includes); octopus_entity ent = ring_leader.issue_identity(); octopus_request_id req_id(ent, 1); outcome start_ret = ring_leader.evaluate(initiate, req_id); if (start_ret != tentacle::OKAY) - RETURN_ERROR_RFC("failed to start the comparison", NONE_READY); + RETURN_ERROR_RFC("failed to build target tree", NONE_READY); file_transfer_infoton *reply_from_init = (file_transfer_infoton *)ring_leader.acquire_specific_result(req_id); if (!reply_from_init) { LOG("spewing list of what IS there..."); LOG(ring_leader.responses().text_form()); - RETURN_ERROR_RFC("no response to tree compare start", NONE_READY); + RETURN_ERROR_RFC("no response to request to build target tree", NONE_READY); + } + + if (reply_from_init->_success != OKAY) { + RETURN_ERROR_RFC("failed to get good result from building target tree", reply_from_init->_success); + } + +//now repeating a lot of above to get tree compare going. + +//hmmm: this kind of object creation should be packaged in helper functions. + file_transfer_infoton *comparison_req = new file_transfer_infoton; + comparison_req->_request = true; + comparison_req->_command = file_transfer_infoton::TREE_COMPARISON; + comparison_req->_src_root = source_root; + comparison_req->_dest_root = target_dir; + // make a directory snapshot with just directories, no files. + directory_tree target_area(target_dir); + +//hmmm: simple asset counting debugging in calculate would be nice too. + target_area.calculate( !(transfer_mode & file_transfer_tentacle::COMPARE_CONTENT_SAMPLE) ); + + comparison_req->package_tree_info(target_area, includes); + + ent = ring_leader.issue_identity(); + req_id = octopus_request_id(ent, 1); + start_ret = ring_leader.evaluate(comparison_req, req_id); + if (start_ret != tentacle::OKAY) + RETURN_ERROR_RFC("failed to build target tree", NONE_READY); + + reply_from_init = (file_transfer_infoton *)ring_leader.acquire_specific_result(req_id); + if (!reply_from_init) { +LOG("spewing list of what IS there..."); +LOG(ring_leader.responses().text_form()); + RETURN_ERROR_RFC("no response to request to build target tree", NONE_READY); } + +//resuming + + filename_list diffs; byte_array pack_copy = reply_from_init->_packed_data; if (!diffs.unpack(pack_copy)) {