From 8199ae70dd874c530f4faf928b70e7dfd3f592a1 Mon Sep 17 00:00:00 2001 From: "Fred T. Hamster" Date: Mon, 16 Feb 2026 19:49:38 -0500 Subject: [PATCH] getting the octopus tests back online some progress has been made. test bin threaded now has a cool printout for adding and removing entity data. also it no longer crashes (as much?). --- nucleus/library/loggers/console_logger.cpp | 20 +- octopi/library/octopus/entity_data_bin.cpp | 36 ++- octopi/library/octopus/entity_data_bin.h | 2 +- octopi/library/tests_cromp/makefile | 2 +- octopi/library/tests_octopus/makefile | 2 +- .../tests_octopus/test_bin_threaded.cpp | 218 +++++++++++------- .../tests_octopus/test_file_transfer.cpp | 51 ++-- 7 files changed, 198 insertions(+), 133 deletions(-) diff --git a/nucleus/library/loggers/console_logger.cpp b/nucleus/library/loggers/console_logger.cpp index aeb304b0..69e28604 100644 --- a/nucleus/library/loggers/console_logger.cpp +++ b/nucleus/library/loggers/console_logger.cpp @@ -15,9 +15,12 @@ #include "console_logger.h" #include "logging_filters.h" +#include + #include using namespace basis; +using namespace textual; namespace loggers { @@ -33,28 +36,15 @@ if (filter) {} //temp ignored FILE *log_to = stdout; if (c_target == TO_STDERR) log_to = stderr; - -//hmmm: temp simplified form during bootup of new hoople. -fprintf(log_to, "%s\n", (char *)info.observe()); - -/* -hmmm: need filter set support! if (member(filter)) { -*/ // format the output with %s to ensure we get all characters, rather // than having some get interpreted if we used info as the format spec. -// fprintf(log_to, "%s", (char *)info.s()); + fprintf(log_to, "%s", (char *)info.observe()); // send the EOL char if the style is appropriate for that. -// if (eol() != NO_ENDING) fprintf(log_to, "%s", get_ending().s()); - - + if (eol() != parser_bits::NO_ENDING) fprintf(log_to, "%s", get_ending().s()); // write immediately to avoid lost output on crash. fflush(log_to); - -/* -hmmm: need filter set support! } -*/ return common::OKAY; } diff --git a/octopi/library/octopus/entity_data_bin.cpp b/octopi/library/octopus/entity_data_bin.cpp index a1584e0d..90898289 100644 --- a/octopi/library/octopus/entity_data_bin.cpp +++ b/octopi/library/octopus/entity_data_bin.cpp @@ -234,20 +234,29 @@ bool entity_data_bin::add_item(infoton *to_add, bask->_last_active = time_stamp(); // reset activity time. - // count up the current amount of data in use. - int current_size = 0; - for (int i = 0; i < bask->elements(); i++) - current_size += bask->borrow(i)->_item->packed_size(); - + // count the current amount of data in use. + int current_count = 0; int current_size = 0; + bool worked = get_sizes(orig_id._entity, current_count, current_size); +#ifdef DEBUG_ENTITY_DATA_BIN +// LOG(a_sprintf("size before add=%d", current_size)); +#endif if (current_size + to_add->packed_size() > _max_per_ent) { +#ifdef DEBUG_ENTITY_DATA_BIN + LOG(a_sprintf("size limit would be exceeded if we stored this product (would grow to %d with limit of %d).", current_size + to_add->packed_size(), _max_per_ent)); +#endif WHACK(holder); -LOG(astring("size limit would be exceeded if we stored this product")); return false; } // append the latest production to the list. bask->append(holder); _items_held++; + +#ifdef DEBUG_ENTITY_DATA_BIN +// worked = get_sizes(orig_id._entity, current_count, current_size); +// LOG(a_sprintf("size after add=%d", current_size)); +#endif + return true; } @@ -277,6 +286,12 @@ infoton *entity_data_bin::acquire_for_any(octopus_request_id &id) _table->apply(any_item_applier, &apple); if (!apple._any_item) return NULL_POINTER; DUMP_STATE; +#ifdef DEBUG_ENTITY_DATA_BIN +// int current_count = 0; +// int current_size = 0; +// bool worked = get_sizes(id._entity, current_count, current_size); +// LOG(a_sprintf("size before remove=%d", current_size)); +#endif // retrieve the information from our basket that was provided. infoton_holder *found = apple._any_item->acquire(0); apple._any_item->zap(0, 0); @@ -293,10 +308,16 @@ infoton *entity_data_bin::acquire_for_any(octopus_request_id &id) found->_item = NULL_POINTER; // clear so it won't be whacked. WHACK(found); _items_held--; + //#ifdef DEBUG_ENTITY_DATA_BIN if (_items_held < 0) LOG("logic error: number of items went below zero."); //#endif + +#ifdef DEBUG_ENTITY_DATA_BIN +// worked = get_sizes(id._entity, current_count, current_size); +// LOG(a_sprintf("size after remove=%d", current_size)); +#endif return to_return; } @@ -469,8 +490,7 @@ void entity_data_bin::clean_out_deadwood(int decay_interval) } } -bool entity_data_bin::get_sizes(const octopus_entity &id, int &items, - int &bytes) +bool entity_data_bin::get_sizes(const octopus_entity &id, int &items, int &bytes) const { FUNCDEF("get_sizes"); items = 0; diff --git a/octopi/library/octopus/entity_data_bin.h b/octopi/library/octopus/entity_data_bin.h index 0ed4b004..d4776193 100644 --- a/octopi/library/octopus/entity_data_bin.h +++ b/octopi/library/octopus/entity_data_bin.h @@ -64,7 +64,7 @@ public: // the lock on the bin, but if it's zero then that's a good reason to // avoid looking for data yet. - bool get_sizes(const octopus_entity &id, int &items, int &bytes); + bool get_sizes(const octopus_entity &id, int &items, int &bytes) const; // finds the storage for "id". if there is any there, true is returned // and "items" is set to the number of pending items and "bytes" is set // to the number of bytes for those items. diff --git a/octopi/library/tests_cromp/makefile b/octopi/library/tests_cromp/makefile index ab8f75a4..53353559 100644 --- a/octopi/library/tests_cromp/makefile +++ b/octopi/library/tests_cromp/makefile @@ -6,7 +6,7 @@ PROJECT = test_cromp TYPE = test TARGETS = test_cromp_client.exe test_cromp_server.exe test_many_cromp.exe LOCAL_LIBS_USED = unit_test crypto application configuration filesystem loggers \ - mathematics nodes processes sockets structures textual timely octopus cromp \ + mathematics nodes processes sockets structures textual timely cromp octopus \ tentacles structures basis USE_SSL = t LAST_TARGETS = create_decoder_ring run_client_server_test diff --git a/octopi/library/tests_octopus/makefile b/octopi/library/tests_octopus/makefile index ba9b2123..138da6de 100644 --- a/octopi/library/tests_octopus/makefile +++ b/octopi/library/tests_octopus/makefile @@ -7,7 +7,7 @@ TYPE = test TARGETS = test_bin.exe test_bin_threaded.exe test_entity.exe test_identity.exe \ test_security.exe test_unpacker.exe test_file_transfer.exe LOCAL_LIBS_USED = tentacles octopus sockets unit_test application configuration loggers \ - textual timely processes filesystem structures basis + textual octopus timely configuration nodes processes filesystem structures basis #VCPP_USE_SOCK = t RUN_TARGETS = $(ACTUAL_TARGETS) diff --git a/octopi/library/tests_octopus/test_bin_threaded.cpp b/octopi/library/tests_octopus/test_bin_threaded.cpp index f069fe8d..ff786f2d 100644 --- a/octopi/library/tests_octopus/test_bin_threaded.cpp +++ b/octopi/library/tests_octopus/test_bin_threaded.cpp @@ -12,14 +12,14 @@ * Please send any updates to: fred@gruntose.com * \*****************************************************************************/ -#include #include +#include #include -#include #include #include #include #include +#include #include #include #include @@ -52,10 +52,43 @@ using namespace textual; using namespace timely; using namespace unit_test; +// our macros for logging (with or without a timestamp). +#define LOG(s) CLASS_EMERGENCY_LOG(program_wide_logger::get(), astring(s)) + +// the base log feature just prints the text to the console with no carriage return or extra flair. +// it does count up how many characters have been printed though, and does an EOL when it seems like it would be reasonable (80 chars-ish). +// note that this code makes no attempt to worry about any other printing that's happening; it has a very egocentric view of what's on the +// terminal so far. +static int chars_printed = 0; +const int MAXIMUM_CHARS_PER_LINE = 108; +//hmmm: may want to make the line size selectable, if we keep some version of this code around. +SAFE_STATIC(console_logger, ted, ); +SAFE_STATIC(mutex, __teds_lock, ) +#define BASE_LOG(s) { \ + auto_synchronizer critical_section(__teds_lock()); \ + /* we set the eol every time, since console_logger constructor doesn't currently provide. */ \ + ted().eol(parser_bits::NO_ENDING); \ + astring joe(s); \ + int len = joe.length(); \ + /* naive check for line length. */ \ + if (chars_printed + len > MAXIMUM_CHARS_PER_LINE) { \ + ted().log(astring("\n"), basis::ALWAYS_PRINT); \ + chars_printed = 0; \ + } \ + chars_printed += len; \ + ted().log(joe, basis::ALWAYS_PRINT); \ +} + // global constants... +//const int DEFAULT_RUN_TIME = 80 * MINUTE_ms; +//const int DEFAULT_RUN_TIME = 2 * MINUTE_ms; +//const int DEFAULT_RUN_TIME = 28 * SECOND_ms; +const int DEFAULT_RUN_TIME = 4 * SECOND_ms; + // the length of time to run the program. + // how much data is the entity data bin allowed to hold at one time. -const int MAXIMUM_DATA_PER_ENTITY = 1 * KILOBYTE; +const int MAXIMUM_DATA_PER_ENTITY = 5 * KILOBYTE; //tiny limit to test having too much data. // controls the timing of the thread that adds items. @@ -63,6 +96,7 @@ const int MIN_ADDER_THREAD_PAUSE = 3; const int MAX_ADDER_THREAD_PAUSE = 20; // controls the timing of the item deleting thread. +// we currently have this biased to be slower than the adder, so things accumulate. const int MIN_WHACKER_THREAD_PAUSE = 8; const int MAX_WHACKER_THREAD_PAUSE = 70; @@ -70,38 +104,29 @@ const int MAX_WHACKER_THREAD_PAUSE = 70; const int MIN_TIDIER_THREAD_PAUSE = 60; const int MAX_TIDIER_THREAD_PAUSE = 500; -// monk is kept asleep most of the time or he'd be trashing -// all our data too frequently. -const int MIN_MONK_THREAD_PAUSE = 2 * MINUTE_ms; -const int MAX_MONK_THREAD_PAUSE = 4 * MINUTE_ms; +// monk is kept asleep most of the time or he'd be trashing all our data too frequently. +const int MIN_MONK_THREAD_PAUSE = 42 * SECOND_ms; +const int MAX_MONK_THREAD_PAUSE = 64 * SECOND_ms; -// the range of new items added whenever the creator thread is hit. -const int MINIMUM_ITEMS_ADDED = 1; -const int MAXIMUM_ITEMS_ADDED = 20; +// the range of new items added whenever the creator or destroyer threads are hit. +const int MINIMUM_ITEMS_HANDLED = 1; +const int MAXIMUM_ITEMS_HANDLED = 20; const int DEFAULT_THREADS = 90; // the number of threads we create by default. -const int DEFAULT_RUN_TIME = 80 * MINUTE_ms; -//2 * MINUTE_ms; - // the length of time to run the program. - const int DATA_DECAY_TIME = 1 * MINUTE_ms; // how long we retain unclaimed data. const int MONKS_CLEANING_TIME = 10 * SECOND_ms; // a very short duration for data to live. -#define LOG(to_print) printf("%s\n", (char *)astring(to_print).s()); -//CLASS_EMERGENCY_LOG(program_wide_logger::get().get(), to_print) - // our macro for logging with a timestamp. - // global objects... chaos _rando; // our randomizer. -// replace app_shell version with local randomizer, so all the static -// functions can employ it also. +/* replaces app_shell version with local randomizer, so all the static +functions can employ it also. */ #define randomizer() _rando entity_data_bin binger(MAXIMUM_DATA_PER_ENTITY); @@ -130,10 +155,11 @@ octopus_request_id create_request_id() } // this thread creates new items for the entity data bin. +// also known as the adder. class ballot_box_stuffer : public ethread { public: - ballot_box_stuffer() : ethread(0) { + ballot_box_stuffer() : ethread(MIN_ADDER_THREAD_PAUSE) { FUNCDEF("constructor"); LOG("+creator"); } @@ -147,37 +173,41 @@ public: void perform_activity(void *formal(data)) { FUNCDEF("perform_activity"); - while (!should_stop()) { - // add a new item to the cache. - int how_many = randomizer().inclusive(MINIMUM_ITEMS_ADDED, - MAXIMUM_ITEMS_ADDED); - for (int i = 0; i < how_many; i++) { - string_array random_strings; - int string_count = randomizer().inclusive(1, 10); - // we create a random classifier, just to use up some space. - for (int q = 0; q < string_count; q++) { - random_strings += string_manipulation::make_random_name(); - } - unhandled_request *newbert = new unhandled_request(create_request_id(), - random_strings); - binger.add_item(newbert, create_request_id()); + // add a new item to the cache. + int how_many = randomizer().inclusive(MINIMUM_ITEMS_HANDLED, + MAXIMUM_ITEMS_HANDLED); + for (int i = 0; i < how_many; i++) { + string_array random_strings; + int string_count = randomizer().inclusive(1, 10); + // we create a random classifier, just to use up some space. + for (int q = 0; q < string_count; q++) { + random_strings += string_manipulation::make_random_name(); } - // snooze. - int sleepy_time = randomizer().inclusive(MIN_ADDER_THREAD_PAUSE, - MAX_ADDER_THREAD_PAUSE); - time_control::sleep_ms(sleepy_time); + unhandled_request *newbert = new unhandled_request(create_request_id(), + random_strings); + BASE_LOG("+"); + binger.add_item(newbert, create_request_id()); } - } + // snooze. + int sleepy_time = randomizer().inclusive(MIN_ADDER_THREAD_PAUSE, + MAX_ADDER_THREAD_PAUSE); + time_control::sleep_ms(sleepy_time); + // reset the thread's snooze timing. + ethread::sleep_time(sleepy_time); + } }; +////////////// + // this thread eliminates entries in the ballot box. +// also known as the whacker. class vote_destroyer : public ethread { public: - vote_destroyer() : ethread(0) { + vote_destroyer() : ethread(MIN_WHACKER_THREAD_PAUSE) { FUNCDEF("constructor"); - LOG("+destroyer"); + BASE_LOG("+destroyer"); } virtual ~vote_destroyer() { @@ -189,16 +219,22 @@ public: void perform_activity(void *formal(data)) { FUNCDEF("perform_activity"); - while (!should_stop()) { + int how_many = randomizer().inclusive(MINIMUM_ITEMS_HANDLED, + MAXIMUM_ITEMS_HANDLED); + for (int i = 0; i < how_many; i++) { // snag any old item and drop it on the floor. octopus_request_id id; infoton *found = binger.acquire_for_any(id); + if (!found) break; // nothing to whack there. + BASE_LOG("-"); WHACK(found); - // snooze. - int sleepy_time = randomizer().inclusive(MIN_WHACKER_THREAD_PAUSE, - MAX_WHACKER_THREAD_PAUSE); - time_control::sleep_ms(sleepy_time); } + // snooze. + int sleepy_time = randomizer().inclusive(MIN_WHACKER_THREAD_PAUSE, + MAX_WHACKER_THREAD_PAUSE); + time_control::sleep_ms(sleepy_time); + // re-schedule the thread. + ethread::sleep_time(sleepy_time); } }; @@ -220,14 +256,13 @@ public: void perform_activity(void *formal(data)) { FUNCDEF("perform_activity"); - while (!should_stop()) { - // make sure there's nothing rotting too long. - binger.clean_out_deadwood(DATA_DECAY_TIME); - // snooze. - int sleepy_time = randomizer().inclusive(MIN_TIDIER_THREAD_PAUSE, - MAX_TIDIER_THREAD_PAUSE); - time_control::sleep_ms(sleepy_time); - } + // make sure there's nothing rotting too long. + binger.clean_out_deadwood(DATA_DECAY_TIME); + // snooze. + int sleepy_time = randomizer().inclusive(MIN_TIDIER_THREAD_PAUSE, + MAX_TIDIER_THREAD_PAUSE); + time_control::sleep_ms(sleepy_time); + ethread::sleep_time(sleepy_time); } }; @@ -249,30 +284,29 @@ public: void perform_activity(void *formal(data)) { FUNCDEF("perform_activity"); - while (!should_stop()) { - { - // one activation of monk has devastating consequences. we empty out - // the data one item at a time until we see no data at all. after - // cleaning each item, we ensure that the deadwood is cleaned out. -//// binger._ent_lock->lock(); - auto_synchronizer l(binger.locker()); -LOG(a_sprintf("monk sees %d items.", binger.items_held())); - while (binger.items_held()) { - // grab one instance of any item in the bin. - octopus_request_id id; - infoton *found = binger.acquire_for_any(id); - WHACK(found); - // also clean out things a lot faster than normal. - binger.clean_out_deadwood(MONKS_CLEANING_TIME); - } -/// binger._ent_lock->unlock(); - } -LOG(a_sprintf("after a little cleaning, monk sees %d items.", binger.items_held())); - // snooze. - int sleepy_time = randomizer().inclusive(MIN_MONK_THREAD_PAUSE, - MAX_MONK_THREAD_PAUSE); - time_control::sleep_ms(sleepy_time); + // one activation of monk has devastating consequences. we empty out + // the data one item at a time until we see no data at all. after + // cleaning each item, we ensure that the deadwood is cleaned out. + auto_synchronizer l(binger.locker()); +LOG(a_sprintf("monk sees %d items and will clean them all.", binger.items_held())); + while (binger.items_held()) { + // grab one instance of any item in the bin. + octopus_request_id id; + infoton *found = binger.acquire_for_any(id); + if (!found) break; // nothing to see here. + BASE_LOG("-"); + WHACK(found); + // also clean out things a lot faster than normal. + binger.clean_out_deadwood(MONKS_CLEANING_TIME); } +LOG(a_sprintf("after a little light cleaning, monk sees %d items.", binger.items_held())); + // snooze. + int sleepy_time = randomizer().inclusive(MIN_MONK_THREAD_PAUSE, MAX_MONK_THREAD_PAUSE); + // reschedule the thread for the new snooze. and note how we are not actually stuck waiting + // for the whole sleep time, given how ethread works with timed threads. an old implementation + // actually slept uninterruptably for the whole snooze time, which was really off-putting and + // rude. + ethread::sleep_time(sleepy_time); } }; @@ -292,6 +326,17 @@ int test_entity_data_bin_threaded::execute() { FUNCDEF("execute"); + int duration = DEFAULT_RUN_TIME; + if (application::_global_argc >= 2) { + astring duration_string = application::_global_argv[1]; + if (duration_string.length()) { + duration = duration_string.convert(DEFAULT_RUN_TIME); + LOG(a_sprintf("user specified runtime duration of %d seconds.", duration)); + // convert from seconds to milliseconds. + duration *= SECOND_ms; + } + } + amorph thread_list; for (int i = 0; i < DEFAULT_THREADS; i++) { @@ -314,20 +359,17 @@ int test_entity_data_bin_threaded::execute() thread_list[thread_list.elements() - 1]->start(NULL_POINTER); } - time_stamp when_to_leave(DEFAULT_RUN_TIME); + time_stamp when_to_leave(duration); while (when_to_leave > time_stamp()) { time_control::sleep_ms(100); } -// LOG("now cancelling all threads...."); - -// for (int j = 0; j < thread_list.elements(); j++) thread_list[j]->cancel(); - -// LOG("now stopping all threads...."); - -// for (int k = 0; k < thread_list.elements(); k++) thread_list[k]->stop(); - -// LOG("resetting thread list...."); +//hmmm: this code shouldn't be needed! thread cabinet should do it!!!! +///LOG("now cancelling all threads...."); +///for (int j = 0; j < thread_list.elements(); j++) thread_list[j]->cancel(); +///LOG("now stopping all threads...."); +///for (int k = 0; k < thread_list.elements(); k++) thread_list[k]->stop(); +///LOG("resetting thread list...."); thread_list.reset(); // should whack all threads. diff --git a/octopi/library/tests_octopus/test_file_transfer.cpp b/octopi/library/tests_octopus/test_file_transfer.cpp index 9deb90a7..9b99816e 100644 --- a/octopi/library/tests_octopus/test_file_transfer.cpp +++ b/octopi/library/tests_octopus/test_file_transfer.cpp @@ -16,20 +16,33 @@ * Please send any updates to: fred@gruntose.com * \*****************************************************************************/ +#include +#include #include -#include -#include #include -#include +#include +#include #include #include - -#define LOG(s) CLASS_EMERGENCY_LOG(program_wide_logger::get(), s) - -class test_file_transfer_tentacle : public application_shell +#include + +using namespace application; +using namespace basis; +//using namespace configuration; +using namespace loggers; +using namespace mathematics; +using namespace octopi; +//using namespace sockets; +using namespace structures; +using namespace textual; +using namespace unit_test; + +#define LOG(s) CLASS_EMERGENCY_LOG(program_wide_logger::get(), astring(s)) + +class test_file_transfer_tentacle : virtual public unit_base, virtual public application_shell { public: - test_file_transfer_tentacle() : application_shell(static_class_name()) {} + test_file_transfer_tentacle() : application_shell() {} DEFINE_CLASS_NAME("test_dirtree_fcopy"); int execute(); }; @@ -38,8 +51,8 @@ int test_file_transfer_tentacle::execute() { FUNCDEF("execute"); - if (__argc < 3) { - log("\ + if (application::_global_argc < 3) { + LOG("\ This program needs two parameters:\n\ a directory for the source root and one for the target root.\n\ Optionally, a third parameter may specify a starting point within the\n\ @@ -49,18 +62,18 @@ files to include; only they will be transferred.\n"); return 23; } - astring source_dir = __argv[1]; - astring target_dir = __argv[2]; + astring source_dir = application::_global_argv[1]; + astring target_dir = application::_global_argv[2]; astring source_start = ""; - if (__argc >= 4) { - source_start = __argv[3]; + if (application::_global_argc >= 4) { + source_start = application::_global_argv[3]; } string_array includes; - if (__argc >= 5) { - for (int i = 4; i < __argc; i++) { - includes += __argv[i]; + if (application::_global_argc >= 5) { + for (int i = 4; i < application::_global_argc; i++) { + includes += application::_global_argv[i]; } } @@ -175,10 +188,10 @@ LOG(a_sprintf("size in array now: %d", copy.length())); */ if (returned == common::OKAY) - guards::alert_message("file_transfer_tentacle:: works for those " + critical_events::alert_message("file_transfer_tentacle:: works for those " "functions tested."); else - guards::alert_message(astring("file_transfer_tentacle:: failed with " + critical_events::alert_message(astring("file_transfer_tentacle:: failed with " "outcome=") + recursive_file_copy::outcome_name(returned)); return 0; } -- 2.43.0