[Solved] Results of the forecasting performance operator seems illogical

Dear all,

I found several entries in the forum regarding this topic but it is still not clear to me what this operator really does.
In this post for example there is a description: http://rapid-i.com/rapidforum/index.php/topic,2680.0.html

In the attached sample process I create a label attribute with the windowing operator from att2 and feed this into a validation. Here is a fictious testing example set:

label(att2)     prediction     att1     att2     att3
             4                   5         8         1         2
             7                   9         2         4         0

I would assume that only one row is necessary to determine the trend per iteration.
(e.g. label(4)>att2(1) AND prediction(5)>att2(1) --> prediction trend should be correct)
But it seems to work in a different way...

- How does the forecasting performance operator actually calculates the performance?
- How does the performance operator know which attribute has been forecasted by the label, e.g. att2?
(this cannot be set in the parameters)
- Why does the operator need at least a set of two rows - if test window width is set to 1 the result is "unkown"?
(windowing has already been done in advance so that the label is yet shifted and no second row would be needed)
- What influence has the parameter "main criterion"?


Please advise...


Best regards
Sachs



The attached sample shows a simple process which is supposed to train a model on forecasting time series.
The first breakpoint shows the validation's testset after the model made its prediction.
The second breakpoint shows the corresponding performance of the model for this specific testset.

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.2.008">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="5.2.008" expanded="true" name="Process">
    <process expanded="true" height="502" width="547">
      <operator activated="true" class="subprocess" compatibility="5.2.008" expanded="true" height="76" name="Generate Data" width="90" x="45" y="30">
        <process expanded="true" height="383" width="299">
          <operator activated="true" class="generate_data" compatibility="5.2.008" expanded="true" height="60" name="Generate Data (2)" width="90" x="45" y="30">
            <parameter key="number_examples" value="30"/>
            <parameter key="number_of_attributes" value="3"/>
            <parameter key="attributes_lower_bound" value="0.0"/>
          </operator>
          <operator activated="true" class="select_attributes" compatibility="5.2.008" expanded="true" height="76" name="Select Attributes" width="90" x="179" y="30">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="label"/>
            <parameter key="invert_selection" value="true"/>
            <parameter key="include_special_attributes" value="true"/>
          </operator>
          <connect from_op="Generate Data (2)" from_port="output" to_op="Select Attributes" to_port="example set input"/>
          <connect from_op="Select Attributes" from_port="example set output" to_port="out 1"/>
          <portSpacing port="source_in 1" spacing="0"/>
          <portSpacing port="sink_out 1" spacing="0"/>
          <portSpacing port="sink_out 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="series:windowing" compatibility="5.2.000" expanded="true" height="76" name="Windowing" width="90" x="179" y="30">
        <parameter key="horizon" value="1"/>
        <parameter key="window_size" value="1"/>
        <parameter key="create_label" value="true"/>
        <parameter key="label_attribute" value="att2"/>
      </operator>
      <operator activated="true" class="series:sliding_window_validation" compatibility="5.2.000" expanded="true" height="112" name="Validation" width="90" x="313" y="30">
        <parameter key="training_window_width" value="5"/>
        <parameter key="training_window_step_size" value="1"/>
        <parameter key="test_window_width" value="2"/>
        <process expanded="true" height="502" width="165">
          <operator activated="true" class="support_vector_machine_linear" compatibility="5.2.008" expanded="true" height="76" name="SVM (Linear)" width="90" x="45" y="30"/>
          <connect from_port="training" to_op="SVM (Linear)" to_port="training set"/>
          <connect from_op="SVM (Linear)" from_port="model" to_port="model"/>
          <portSpacing port="source_training" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true" height="502" width="299">
          <operator activated="true" class="apply_model" compatibility="5.2.008" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
            <list key="application_parameters"/>
          </operator>
          <operator activated="true" breakpoints="before,after" class="series:forecasting_performance" compatibility="5.2.000" expanded="true" height="76" name="Performance" width="90" x="179" y="30">
            <parameter key="horizon" value="1"/>
            <parameter key="main_criterion" value="prediction_trend_accuracy"/>
          </operator>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
          <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_averagable 1" spacing="0"/>
          <portSpacing port="sink_averagable 2" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Generate Data" from_port="out 1" to_op="Windowing" to_port="example set input"/>
      <connect from_op="Windowing" from_port="example set output" to_op="Validation" to_port="training"/>
      <connect from_op="Validation" from_port="averagable 1" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>