Record and compare runtime of different algorithms?

Fred12

hi,

I want to compare 3 different learners in a loop for the same dataset, I also want to record the runtime (e.g. in the logfile) for the learners to do the cross validation.. is there some log field that does that for one or all algorithms?

JEdward

The log operator with the operator for the algorithm selected and value set to Execution-Time. 

 

You can do this from either the CrossValidation or from the Algorithm itself, but I would recommend putting it inside your X-Validation so you can take a count of every execution of the chosen algorithm and can later calculate average run-times.

 

Sample process:

 

<?xml version="1.0" encoding="UTF-8"?><process version="7.3.000">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="7.3.000" expanded="true" name="Process">
    <process expanded="true">
      <operator activated="true" class="retrieve" compatibility="7.3.000" expanded="true" height="68" name="Deals" width="90" x="45" y="34">
        <parameter key="repository_entry" value="//Samples/data/Deals"/>
      </operator>
      <operator activated="true" class="generate_id" compatibility="7.3.000" expanded="true" height="82" name="Generate ID" width="90" x="179" y="34"/>
      <operator activated="true" class="concurrency:cross_validation" compatibility="7.3.000" expanded="true" height="145" name="Cross Validation" width="90" x="313" y="34">
        <parameter key="number_of_folds" value="3"/>
        <parameter key="sampling_type" value="linear sampling"/>
        <process expanded="true">
          <operator activated="true" class="parallel_decision_tree" compatibility="7.3.000" expanded="true" height="82" name="Decision Tree" width="90" x="112" y="34"/>
          <operator activated="true" class="log" compatibility="7.3.000" expanded="true" height="82" name="Log" width="90" x="246" y="34">
            <list key="log">
              <parameter key="Decision Tree Execution Number" value="operator.Decision Tree.value.applycount"/>
              <parameter key="Decision Tree Execution Time" value="operator.Decision Tree.value.execution-time"/>
            </list>
          </operator>
          <connect from_port="training set" to_op="Decision Tree" to_port="training set"/>
          <connect from_op="Decision Tree" from_port="model" to_op="Log" to_port="through 1"/>
          <connect from_op="Log" from_port="through 1" to_port="model"/>
          <portSpacing port="source_training set" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true">
          <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model" width="90" x="45" y="34">
            <list key="application_parameters"/>
          </operator>
          <operator activated="true" class="performance_classification" compatibility="7.3.000" expanded="true" height="82" name="Performance" width="90" x="179" y="34">
            <list key="class_weights"/>
          </operator>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
          <connect from_op="Performance" from_port="performance" to_port="performance 1"/>
          <connect from_op="Performance" from_port="example set" to_port="test set results"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_test set results" spacing="0"/>
          <portSpacing port="sink_performance 1" spacing="0"/>
          <portSpacing port="sink_performance 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="log_to_data" compatibility="7.3.000" expanded="true" height="82" name="Log to Data" width="90" x="514" y="187"/>
      <operator activated="true" class="aggregate" compatibility="7.3.000" expanded="true" height="82" name="Aggregate" width="90" x="648" y="187">
        <parameter key="default_aggregation_function" value="standard_deviation"/>
        <list key="aggregation_attributes">
          <parameter key="Decision Tree Execution Time" value="average"/>
          <parameter key="Decision Tree Execution Time" value="standard_deviation"/>
        </list>
      </operator>
      <connect from_op="Deals" from_port="output" to_op="Generate ID" to_port="example set input"/>
      <connect from_op="Generate ID" from_port="example set output" to_op="Cross Validation" to_port="example set"/>
      <connect from_op="Cross Validation" from_port="model" to_port="result 1"/>
      <connect from_op="Cross Validation" from_port="performance 1" to_port="result 2"/>
      <connect from_op="Log to Data" from_port="exampleSet" to_op="Aggregate" to_port="example set input"/>
      <connect from_op="Aggregate" from_port="example set output" to_port="result 3"/>
      <connect from_op="Aggregate" from_port="original" to_port="result 4"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
      <portSpacing port="sink_result 3" spacing="0"/>
      <portSpacing port="sink_result 4" spacing="0"/>
      <portSpacing port="sink_result 5" spacing="0"/>
    </process>
  </operator>
</process>