Commit 26adb5d8 authored by Oliver Heidmann's avatar Oliver Heidmann
Browse files

merged branch argument_rework

parents 0a78bd58 0bfddbc6
Pipeline #11262 passed with stages
in 16 minutes and 4 seconds
......@@ -234,7 +234,7 @@ Seltime(void *process)
if (redirectFound && redirectEnabled)
{
Debug(Yellow("Redirecting to %s"), newCommand);
((Process *) process)->init_process("select", newCommand.c_str());
((Process *) process)->init_process("select", {newCommand});
return Select(process);
// If a redirect was found the entire process is ended through this return!
}
......
......@@ -1538,7 +1538,7 @@ main(int argc, char *argv[])
close_library_handles();
#endif
g_processManager.create_processes(new_argv.size(), (const char **) &new_cargv[0]);
g_processManager.create_processes(new_argv.size(), new_argv);
timer_start(timer_total);
g_processManager.run_processes();
timer_stop(timer_total);
......
......@@ -68,6 +68,27 @@ module_t::toString()
return desc;
}
void
extract_name_and_argument(const std::string &command, std::string &operatorName, std::string &operatorArgument)
{
std::string delimiter = ",";
size_t start = 0;
if (command[0] == '-') start = 1;
size_t len = command.find(delimiter);
if (len == std::string::npos)
{
len = command.size();
operatorArgument = "";
}
else{
operatorArgument = command.substr(len + 1, std::string::npos);
}
operatorName = command.substr(start, len - start);
}
std::string
extract_operator_name(const std::string &operatorCommand)
{
......@@ -76,7 +97,10 @@ extract_operator_name(const std::string &operatorCommand)
const size_t start = (operatorCommand[0] == '-') ? 1 : 0;
size_t len = operatorCommand.find(delimiter);
if (len == std::string::npos) len = operatorCommand.size();
if (len == std::string::npos)
{
len = operatorCommand.size();
}
std::string oper_name = operatorCommand.substr(start, len - start);
return oper_name;
......@@ -89,7 +113,7 @@ is_alias(const std::string &operatorName)
}
std::string
get_original(const char *operatorName)
get_original(const std::string &operatorName)
{
std::string name = extract_operator_name(operatorName);
return (is_alias(name) ? get_aliases()[name] : name);
......@@ -684,5 +708,5 @@ find_module(const std::string &operator_name)
module_t &
get_module(const std::string &operator_name)
{
return get_modules()[get_module_name_to(get_original(operator_name.c_str()))];
return get_modules()[get_module_name_to(get_original(operator_name))];
}
......@@ -51,8 +51,9 @@ struct module_t
std::vector<Alias> aliases;
ModuleRestrictions restrictions = NoRestriction;
module_t(const std::string &mod_name, void *(*p_func)(void *), const char **p_help, const std::vector<std::string> &p_opers, short p_m,
short p_n, short p_siC, short p_soC, ModuleRestrictions p_onlyFirst, const std::vector<Alias> &p_aliases = {});
module_t(const std::string &mod_name, void *(*p_func)(void *), const char **p_help, const std::vector<std::string> &p_opers,
short p_m, short p_n, short p_siC, short p_soC, ModuleRestrictions p_onlyFirst,
const std::vector<Alias> &p_aliases = {});
module_t(){};
std::string toString();
};
......@@ -63,6 +64,7 @@ struct module_t
extern std::vector<void *> custom_modules_lib_handles;
std::string extract_operator_name(const std::string &operatorCommand);
void extract_name_and_argument(const std::string &command, std::string &operatorName, std::string &operatorArgument);
/***
Key: operator alias / Value: operator original name
......@@ -75,8 +77,7 @@ void register_operators(const std::string &p_mod_name, const module_t &p_mod);
std::map<std::string, module_t>::iterator find_module(const std::string &operatorName);
module_t &get_module(const std::string &operatorName);
std::string
get_original(const char *operatorName);
std::string get_original(const std::string &operatorName);
void init_modules();
......
......@@ -44,41 +44,11 @@ set_process_num(int p_num)
processNum = p_num;
}
Process::Process(int p_ID, const std::string &p_operatorName, const char *p_operatorCommand) : m_ID(p_ID), operatorName(p_operatorName)
Process::Process(int p_ID, const std::string &p_operatorName, const std::vector<std::string> &p_arguments)
: m_ID(p_ID), operatorName(p_operatorName)
{
m_isActive = true;
startTime = cdo_get_wtime();
init_process(p_operatorName, p_operatorCommand);
}
void
Process::set_operator_argv(const char *operatorArguments)
{
if (operatorArguments)
{
m_oargv.clear();
Debug(PROCESS && strchr(operatorArguments, ',') != nullptr, "Setting operator arguments: %s", operatorArguments);
std::istringstream f(operatorArguments);
std::string s;
std::string argument = "";
getline(f, s, ',');
while (getline(f, s, ','))
{
argument = s;
while (argument.back() == '\\')
{
argument.back() = ',';
getline(f, s, ',');
argument.append(s);
}
Debug(PROCESS, "Adding argument %s", argument);
m_oargv.push_back(argument);
}
}
init_process(p_operatorName, p_arguments);
}
bool
......@@ -88,20 +58,18 @@ Process::input_is_variable()
}
void
Process::init_process(const std::string &p_operatorName, const char *operatorCommand)
Process::init_process(const std::string &p_operatorName, const std::vector<std::string> &p_arguments)
{
startTime = cdo_get_wtime();
#ifdef HAVE_LIBPTHREAD
threadID = pthread_self();
#endif
m_oargv = p_arguments;
operatorName = get_original(p_operatorName);
set_operator_argv(operatorCommand);
m_operatorCommand = operatorCommand;
m_module = get_module(p_operatorName);
def_prompt(); // has to be called after get operatorName
m_module = get_module(p_operatorName);
// if (m_module.func == nullptr) cdo_abort("Module for operator <%s> not implemented!\n", p_operatorName);
operatorName = std::string(get_original(p_operatorName.c_str()));
}
int
......@@ -460,12 +428,6 @@ Process::has_in_stream(const CdoStreamID p_streamID)
return false;
}
const char *
Process::get_command()
{
return m_operatorCommand + (*m_operatorCommand == '-');
}
size_t
Process::inq_nvals()
{
......@@ -499,7 +461,7 @@ Process::get_oper_argc()
std::string
Process::get_argv(int p_idx)
{
if (!(p_idx > (int)get_oper_argc() && p_idx > 0))
if (!(p_idx > (int) get_oper_argc() && p_idx > 0))
cdo_abort("Process Argv not found. Idx: %d, Process argc: %d", p_idx, m_oargv.size());
return m_oargv[p_idx];
......@@ -510,3 +472,7 @@ Process::get_obase()
{
return m_obase;
}
int Process::get_id(){
return m_ID;
}
......@@ -62,9 +62,9 @@ public:
int ntimesteps = 0;
int m_streamCnt = 0;
char const *m_operatorCommand = "UNINITALIZED";
std::string m_operatorCommand = "UNINITALIZED";
std::string operatorName;
const char *m_obase;
std::string m_obase;
char prompt[64];
short m_noper = 0;
......@@ -74,7 +74,7 @@ public:
std::vector<std::string> m_oargv;
oper_t oper[MAX_OPERATOR];
Process(int p_ID, const std::string &p_operatorNamme, const char *operatorCommand);
Process(int p_ID, const std::string &p_operatorNamme, const std::vector<std::string> &operatorArguments);
pthread_t run();
ProcessStatus m_status = ProcessStatus::Ok;
......@@ -88,11 +88,6 @@ public:
*/
int get_stream_cnt_out();
/**
* returns full command through which this process was created including operator name and operator arguments
*/
const char *get_command();
/**
* Adds a Process as child and creates and adds a new pipe stream.
*/
......@@ -147,9 +142,11 @@ public:
size_t get_oper_argc();
std::string get_argv(int idx);
std::string get_obase();
void set_obase(std::string &obase){m_obase = obase;} //TODO into cc
bool input_is_variable();
int get_id();
void init_process(const std::string &p_operatorName, const char *operatorCommand);
void init_process(const std::string &p_operatorName, const std::vector<std::string> &p_arguments);
private:
Process();
......@@ -158,7 +155,7 @@ private:
* Splits the operator arguments stores them.
* Operator arguments are stored as char arrays and appended with a zero termination.
*/
void set_operator_argv(const char *operatorArguments);
void set_operator_argv(std::string operatorArguments);
};
int get_process_num();
......
......@@ -93,13 +93,14 @@ ProcessManager::clear_processes()
}
const std::shared_ptr<Process>
ProcessManager::create_process_from_command(const char *command)
ProcessManager::create_process_from_command(const std::string &command)
{
Debug(PROCESS_MANAGER, "Creating new process for command: %s with ID: %d ", command, m_numProcesses);
int processID = m_numProcesses++;
const std::string operatorName = extract_operator_name(command);
auto success = m_processes.insert(std::make_pair(processID, std::make_shared<Process>(processID, operatorName, command)));
const std::vector<std::string> arguments = get_operator_argv(command);
auto success = m_processes.insert(std::make_pair(processID, std::make_shared<Process>(processID, operatorName, arguments)));
if (!success.second) cdo_abort("Process %d could not be created", processID);
m_numProcessesActive++;
......@@ -134,7 +135,7 @@ ProcessManager::get_num_active_processes(void)
* as input and exits with error message if not.
*/
const std::shared_ptr<Process>
ProcessManager::create_child_process_for(const std::shared_ptr<Process> &p_parentProcess, const char *argvEntry)
ProcessManager::create_child_process_for(const std::shared_ptr<Process> &p_parentProcess, const std::string &argvEntry)
{
if (p_parentProcess->m_module.restrictions == FilesOnly)
cdo_abort("operator -%s can not be used with pipes.", p_parentProcess->operatorName);
......@@ -144,7 +145,7 @@ ProcessManager::create_child_process_for(const std::shared_ptr<Process> &p_paren
if (newProcess->m_module.streamOutCnt == 0)
cdo_abort("operator -%s can not take -%s with 0 outputs as input", p_parentProcess->operatorName, newProcess->operatorName);
Debug(PROCESS_MANAGER, "Adding new child %s to %s", newProcess->get_command(), p_parentProcess->get_command());
Debug(PROCESS_MANAGER, "Adding new child %s (id: %d) to %s (idL %d)", newProcess->operatorName, newProcess->get_id(), p_parentProcess->operatorName, p_parentProcess->get_id());
p_parentProcess->add_child(newProcess);
newProcess->add_parent(p_parentProcess);
......@@ -153,13 +154,13 @@ ProcessManager::create_child_process_for(const std::shared_ptr<Process> &p_paren
}
std::set<std::string>
ProcessManager::handle_first_operator(int p_argcStart, int argc, const char **argv, const std::shared_ptr<Process> &p_rootProcess)
ProcessManager::handle_first_operator(int p_argcStart, int argc, std::vector<std::string> &argv, const std::shared_ptr<Process> &p_rootProcess)
{
std::set<std::string> files;
for (int i = p_argcStart; i < argc; i++)
{
Debug(PROCESS_MANAGER, "Creating new pstream for output file: %s", argv[i]);
if (strcmp(argv[i], "]") == 0)
if (argv[i] == "]")
{
cdo_abort("missing output file");
}
......@@ -183,7 +184,7 @@ ProcessManager::check_single_bracket_only(const char *p_argvEntry, char p_bracke
}
void
ProcessManager::create_processes(int argc, const char **argv)
ProcessManager::create_processes(int argc, std::vector<std::string> &argv)
{
ParseStatus parseStatus = create_processes_from_input(argc, argv);
if (parseStatus != ParseStatus::Ok)
......@@ -252,12 +253,37 @@ get_stack_as_string(std::stack<std::shared_ptr<Process>> p_stack)
* by our routine (Process::validate) that checks if every process has the
* correct number of inputs and outputs and will throw an error.
*/
std::vector<std::string>
ProcessManager::get_operator_argv(std::string operatorArguments)
{
std::vector<std::string> argument_vector;
Debug(PROCESS && strchr(operatorArguments.c_str(), ',') != nullptr, "Setting operator arguments: %s", operatorArguments);
char delimiter = ',';
size_t pos = operatorArguments.find(delimiter);
if (pos != std::string::npos)
{
// remove operator name
operatorArguments.erase(0, pos + 1);
while ((pos = operatorArguments.find(delimiter)) != std::string::npos)
{
argument_vector.push_back(operatorArguments.substr(0, pos));
Debug("added argument %s", argument_vector.back());
operatorArguments.erase(0, pos + 1);
}
argument_vector.push_back(operatorArguments);
}
return argument_vector;
}
ParseStatus
ProcessManager::create_processes_from_input(int argc, const char **argv)
ProcessManager::create_processes_from_input(int argc, std::vector<std::string> &argv)
{
Debug(PROCESS_MANAGER, "== Process Creation Start ==");
Debug(PROCESS_MANAGER, "operators: %s", argv_to_string(argc, argv));
Debug(PROCESS_MANAGER, "operators: %s", argv_to_string(argv));
const std::shared_ptr<Process> &root_process = create_process_from_command(argv[0]);
int cntOutFiles = (int) root_process->m_module.streamOutCnt;
......@@ -267,7 +293,7 @@ ProcessManager::create_processes_from_input(int argc, const char **argv)
unsigned int lastNonOutputIdx = argc - cntOutFiles;
if (cntOutFiles == -1)
{
root_process->m_obase = argv[argc - 1];
root_process->set_obase(argv[argc - 1]);
cntOutFiles = 1;
lastNonOutputIdx = argc - 1;
}
......@@ -291,7 +317,7 @@ ProcessManager::create_processes_from_input(int argc, const char **argv)
currentProcess = processStack.top();
Debug(PROCESS_MANAGER, "iteration %d, current argv: %s, currentProcess: %s", idx, argv[idx], currentProcess->m_operatorCommand);
argvEntry = argv[idx];
argvEntry = argv[idx].c_str();
//------------------------------------------------------
// case: operator
if (argvEntry[0] == '-')
......
......@@ -39,17 +39,18 @@ private:
int m_numProcesses = 0;
int m_numProcessesActive = 0;
std::set<std::string> handle_first_operator(int p_argcStart, int argc, const char **argv, const std::shared_ptr<Process> &p_rootProcess);
std::set<std::string> handle_first_operator(int p_argcStart, int argc, std::vector<std::string> &argv, const std::shared_ptr<Process> &p_rootProcess);
void check_single_bracket_only(const char *p_argvEntry, char p_bracketType);
void handle_parse_error(ParseStatus p_parseStatus);
int get_num_variable_input_operators();
std::vector<std::string> get_operator_argv(std::string operatorArguments);
const std::shared_ptr<Process> create_process_from_command(const char *p_command);
const std::shared_ptr<Process> create_child_process_for(const std::shared_ptr<Process> &p_parentProcess, const char *p_command);
const std::shared_ptr<Process> create_process_from_command(const std::string &p_command);
const std::shared_ptr<Process> create_child_process_for(const std::shared_ptr<Process> &p_parentProcess, const std::string &p_command);
public:
ParseStatus create_processes_from_input(int argc, const char **argv);
void create_processes(int argc, const char **argv);
ParseStatus create_processes_from_input(int argc, std::vector<std::string> &argv);
void create_processes(int argc, std::vector<std::string> &argv);
void run_processes();
void kill_processes();
void validate_processes();
......
......@@ -331,7 +331,7 @@ cdo_get_command_from_in_stream(int p_streamIndex)
{
return cdo_get_stream_name(p_streamIndex);
}
return process->get_command();
return process->operatorName.c_str(); //TODO
}
bool
......@@ -343,7 +343,7 @@ cdo_assert_files_only()
const char *
cdo_get_obase()
{
return localProcess->m_obase;
return localProcess->get_obase().c_str();
}
void
......@@ -356,7 +356,7 @@ cdo_initialize(void *p_process)
#ifdef HAVE_LIBPTHREAD
localProcess->threadID = pthread_self();
#endif
Debug(PROCESS, "Initializing process: %s", localProcess->m_operatorCommand);
Debug(PROCESS, "Initializing process: %s (id: %d)", localProcess->operatorName , localProcess->get_id());
#ifdef HAVE_LIBPTHREAD
Debug(PROCESS, "process %d thread %ld", localProcess->m_ID, pthread_self());
......
......@@ -31,7 +31,7 @@ go_bandit([]() {
// in1_out1: infile: 1
// in1_out1: infile 2
// outfile: filled by in1_out1
std::vector<const char *> test_argv{ "-in1_out1", "-in2_out1",
std::vector<std::string> test_argv{ "-in1_out1", "-in2_out1",
"-in1_out1", "input_file1",
"-in1_out1", "input_file2",
"output_file" };
......@@ -44,7 +44,7 @@ go_bandit([]() {
// num out
/*clang-format on*/
g_processManager.create_processes_from_input(test_argv.size(), &test_argv[0]);
g_processManager.create_processes_from_input(test_argv.size(), test_argv);
int i;
for (i = 0; i < g_processManager.get_num_processes(); i++)
......@@ -83,7 +83,7 @@ go_bandit([]() {
// in1_out1: infile: 1
// in1_out1: infile 2
// outfile: filled by in1_out1
std::vector<const char *> test_argv{
std::vector<std::string> test_argv{
"-inVariable_out1", "-in2_out1", "-in1_out1", "input_file1",
"-in1_out1", "input_file2", "-in1_out1", "input_file3",
"-in1_out1", "input_file4", "out"
......@@ -93,7 +93,7 @@ go_bandit([]() {
std::vector<unsigned int> expectedOutputs{ 1, 1, 1, 1, 1, 1 };
g_processManager.create_processes_from_input(test_argv.size(),
&test_argv[0]);
test_argv);
int i;
for (i = 0; i < g_processManager.get_num_processes(); i++)
......
......@@ -40,11 +40,11 @@ go_bandit([]() {
//------------------------------------------------------------------------------
bandit::describe("Negative test for unprocessed inputs", [&]() {
/* clang-format off */
std::vector<const char *> argv_unprocessedInput{
std::vector<std::string> argv_unprocessedInput{
"-in2_out1", "-in0_out1", "-in0_out1", "-in0_out1", "out" };
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_unprocessedInput.size(), &argv_unprocessedInput[0]);
argv_unprocessedInput.size(), argv_unprocessedInput);
result_parse = static_cast<int>(parseStatus);
expected_parse = static_cast<int>(ParseStatus::UnprocessedInput);
......@@ -58,12 +58,12 @@ go_bandit([]() {
//------------------------------------------------------------------------------
bandit::describe("Negative test for miss placement of brackets", [&]() {
/* clang-format off */
std::vector<const char *> argv_missingCloseBracket{
std::vector<std::string> argv_missingCloseBracket{
"-in2_out1", "[", "-in0_out1", "-in0_out1", "out"
};
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_missingCloseBracket.size(), &argv_missingCloseBracket[0]);
argv_missingCloseBracket.size(), argv_missingCloseBracket);
result_parse = static_cast<int>(parseStatus);
expected_parse = static_cast<int>(ParseStatus::ClosingBracketMissing);
......@@ -76,12 +76,12 @@ go_bandit([]() {
//------------------------------------------------------------------------------
bandit::describe("Negative test for miss placement of brackets", [&]() {
/* clang-format off */
std::vector<const char *> argv_missingOpenBracket{
std::vector<std::string> argv_missingOpenBracket{
"-in2_out1", "-in0_out1", "-in0_out1", "]", "out"
};
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_missingOpenBracket.size(), &argv_missingOpenBracket[0]);
argv_missingOpenBracket.size(), argv_missingOpenBracket);
result_parse = static_cast<int>(parseStatus);
expected_parse = static_cast<int>(ParseStatus::OpenBracketMissing);
......@@ -94,12 +94,12 @@ go_bandit([]() {
//------------------------------------------------------------------------------
bandit::describe("Negative test for miss placement of brackets", [&]() {
/* clang-format off */
std::vector<const char *> argv_wrongBracketTooMany{
std::vector<std::string> argv_wrongBracketTooMany{
"-in2_out1", "[", "-in0_out1", "-in0_out1", "-in0_out1", "]", "out"
};
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_wrongBracketTooMany.size(), &argv_wrongBracketTooMany[0]);
argv_wrongBracketTooMany.size(), argv_wrongBracketTooMany);
g_processManager.get_process_from_id(0)->check_stream_cnt();
processStatus = g_processManager.get_process_from_id(0)->m_status;
......@@ -118,12 +118,12 @@ go_bandit([]() {
//------------------------------------------------------------------------------
bandit::describe("Negative test for miss placement of brackets", [&]() {
/* clang-format off */
std::vector<const char *> argv_wrongBracketTooFew{
std::vector<std::string> argv_wrongBracketTooFew{
"-in2_out1", "[", "-in0_out1", "]", "out"
};
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_wrongBracketTooFew.size(), &argv_wrongBracketTooFew[0]);
argv_wrongBracketTooFew.size(), argv_wrongBracketTooFew);
g_processManager.get_process_from_id(0)->check_stream_cnt();
processStatus = g_processManager.get_process_from_id(0)->m_status;
......@@ -139,12 +139,12 @@ go_bandit([]() {
//------------------------------------------------------------------------------
bandit::describe("Negative test for unprocessed inputs", [&]() {
/* clang-format off */
std::vector<const char *> argv_missingOutFileOperHasDash{
std::vector<std::string> argv_missingOutFileOperHasDash{
"-in2_out1"};
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_missingOutFileOperHasDash.size(),
&argv_missingOutFileOperHasDash[0]);
argv_missingOutFileOperHasDash);
result_parse = static_cast<int>(parseStatus);
expected_parse = static_cast<int>(ParseStatus::MissingOutFile);
......@@ -155,13 +155,13 @@ go_bandit([]() {
});
bandit::describe("Testing if cdo wildcards with additional unnecessary [] works", [&]() {
/* clang-format off */
std::vector<const char *> argv_unnecessaty_brackets{
std::vector<std::string> argv_unnecessaty_brackets{
"-in2_out1", "[", "[", "-in0_out1", "-in0_out1", "]", "]", "out"
};
/* clang-format on */
parseStatus = g_processManager.create_processes_from_input(
argv_unnecessaty_brackets.size(), &argv_unnecessaty_brackets[0]);
argv_unnecessaty_brackets.size(), argv_unnecessaty_brackets);
g_processManager.get_process_from_id(0)->check_stream_cnt();
processStatus = g_processManager.get_process_from_id(0)->m_status;
......
......@@ -6,8 +6,6 @@
bool is_alias(const std::string &);
std::string get_original(const char *operatorName);
const std::string alias_name = "alias1-1";
void *
testFunction(void *test)
......
......@@ -8,20 +8,78 @@
using namespace snowhouse;
go_bandit([]() {
bandit::describe("Testing operator name extraction from command string", [&]() {
bandit::describe(
"Testing operator name extraction from command string", [&]() {
bandit::it("returns the commandString if it and has no '-' and has no "
"arguments",
[&]() {
AssertThat(extract_operator_name("test"),
Is().EqualTo("test"));
});
bandit::it("returns the commandString if it and has no '-' and has no arguments",
[&]() { AssertThat(extract_operator_name("test"),Is().EqualTo("test")); });
bandit::it("returns the commandString without the '-' while it has no "
"arguments",
[&]() {
AssertThat(extract_operator_name("-test"),
Is().EqualTo("test"));
});
bandit::it("returns the commandString without the '-' while it has no arguments",
[&]() { AssertThat(extract_operator_name("-test"),Is().EqualTo("test")); });
bandit::it(
"returns the commandString without the '-' while it has arguments",
[&]() {
AssertThat(extract_operator_name("-test,arg"),