"Plot all predicitons in SlidingWindowValidation"

wessel

Dear All,

I wish to plot all my predictions made in my SlidingWindowValidation.
Currently I can only plot the predictions made in the FIRST iteration of the SlidingWindowValidation!

To plot the FIRST iteration I have to:
Set a breakpoint in RegressionPerformance.
Click on:
DataTable Tab.
PlotView
Plotter: Series
Index Dimension: Date
Plot series: [single_price, prediction(single_price)]
(A lot of work, would be nice if this could be automated)


Now how can I plot all predictions made in a sliding  my SlidingWindowValidation.
I figured on each iteration I could write the datasets the comes out of ModelApplier to a new dataset.
The problem is CSVExampleSetWriter overwrites the old results.csv on each iteration!
Is there any way to make it append, instead of overwrite?

Regards,

Wessel



Example XML
Yes I know this is not the way to predict single_price.
But this should be the way to evaluate the performance of the prediction.


<operator name="Root" class="Process" expanded="yes">
    <operator name="SalesExampleSetGenerator" class="SalesExampleSetGenerator">
        <parameter key="number_examples" value="1500"/>
    </operator>
    <operator name="FeatureValueTypeFilter" class="FeatureValueTypeFilter">
    </operator>
    <operator name="ChangeAttributeRole" class="ChangeAttributeRole">
        <parameter key="name" value="single_price"/>
        <parameter key="target_role" value="label"/>
    </operator>
    <operator name="SlidingWindowValidation" class="SlidingWindowValidation" expanded="yes">
        <parameter key="training_window_width" value="240"/>
        <parameter key="test_window_width" value="24"/>
        <parameter key="horizon" value="24"/>
        <parameter key="average_performances_only" value="false"/>
        <operator name="LinearRegression" class="LinearRegression">
        </operator>
        <operator name="OperatorChain" class="OperatorChain" expanded="yes">
            <operator name="ModelApplier" class="ModelApplier">
                <parameter key="keep_model" value="true"/>
                <list key="application_parameters">
                </list>
            </operator>
            <operator name="make transaction_id regular" class="ChangeAttributeRole">
                <parameter key="name" value="transaction_id"/>
            </operator>
            <operator name="write to results" class="CSVExampleSetWriter">
                <parameter key="csv_file" value="D:\wessel\Desktop\results.csv"/>
                <parameter key="column_separator" value=","/>
            </operator>
            <operator name="RegressionPerformance" class="RegressionPerformance" breakpoints="after">
                <parameter key="keep_example_set" value="true"/>
                <parameter key="absolute_error" value="true"/>
                <parameter key="correlation" value="true"/>
            </operator>
        </operator>
    </operator>
</operator>

haddock

Hi Wessel,

Use the much unloved process log and his chums, like this...

<operator name="Root" class="Process" expanded="yes">
    <operator name="SalesExampleSetGenerator" class="SalesExampleSetGenerator">
        <parameter key="number_examples"	value="1500"/>
    </operator>
    <operator name="FeatureValueTypeFilter" class="FeatureValueTypeFilter">
    </operator>
    <operator name="ChangeAttributeRole" class="ChangeAttributeRole">
        <parameter key="name"	value="single_price"/>
        <parameter key="target_role"	value="label"/>
    </operator>
    <operator name="SlidingWindowValidation" class="SlidingWindowValidation" expanded="yes">
        <parameter key="keep_example_set"	value="true"/>
        <parameter key="training_window_width"	value="240"/>
        <parameter key="training_window_step_size"	value="1"/>
        <parameter key="test_window_width"	value="1"/>
        <parameter key="horizon"	value="24"/>
        <parameter key="average_performances_only"	value="false"/>
        <operator name="LinearRegression" class="LinearRegression">
        </operator>
        <operator name="OperatorChain" class="OperatorChain" expanded="yes">
            <operator name="ModelApplier" class="ModelApplier">
                <parameter key="keep_model"	value="true"/>
                <list key="application_parameters">
                </list>
            </operator>
            <operator name="RegressionPerformance" class="RegressionPerformance">
                <parameter key="keep_example_set"	value="true"/>
                <parameter key="absolute_error"	value="true"/>
                <parameter key="correlation"	value="true"/>
            </operator>
            <operator name="Data2Log" class="Data2Log">
                <parameter key="attribute_name"	value="single_price"/>
                <parameter key="example_index"	value="-1"/>
            </operator>
            <operator name="Data2Log (2)" class="Data2Log">
                <parameter key="attribute_name"	value="prediction(single_price)"/>
                <parameter key="example_index"	value="-1"/>
            </operator>
            <operator name="ProcessLog" class="ProcessLog">
                <list key="log">
                  <parameter key="single_price"	value="operator.Data2Log.value.data_value"/>
                  <parameter key="prediction(single_price)"	value="operator.Data2Log (2).value.data_value"/>
                </list>
            </operator>
        </operator>
    </operator>
</operator>

wessel

Weird, can't get it to work.
My version is now only logging the last example, of each iteration...

I think this is because your version only works with
<parameter key="test_window_width" value="1"/>
If I change it to 350, it only logs the first 1 in this window.


<operator name="Root" class="Process" expanded="yes">
   <operator name="ExampleSetGenerator" class="ExampleSetGenerator">
       <parameter key="target_function" value="random"/>
       <parameter key="number_examples" value="1500"/>
       <parameter key="number_of_attributes" value="1"/>
   </operator>
   <operator name="remove att1" class="FeatureNameFilter">
       <parameter key="skip_features_with_name" value="att1"/>
   </operator>
   <operator name="IdTagging" class="IdTagging">
   </operator>
   <operator name="regular: id" class="ChangeAttributeRole">
       <parameter key="name" value="id"/>
   </operator>
   <operator name="make sin(id/100)" class="AttributeConstruction">
       <list key="function_descriptions">
         <parameter key="sin(id/100)" value="sin(id/100)"/>
       </list>
   </operator>
   <operator name="window size 100" class="MultivariateSeries2WindowExamples">
   </operator>
   <operator name="label: sin(id/100)-0" class="ChangeAttributeRole">
       <parameter key="name" value="sin(id/100)-0"/>
       <parameter key="target_role" value="label"/>
   </operator>
   <operator name="remove horizon attributes" class="FeatureNameFilter">
       <parameter key="skip_features_with_name" value=".*-([0-9]|[1-4][0-9])"/>
       <parameter key="except_features_with_name" value="id-0"/>
   </operator>
   <operator name="id-0 only" class="FeatureNameFilter">
       <parameter key="skip_features_with_name" value="id-.*"/>
       <parameter key="except_features_with_name" value="id-0"/>
   </operator>
   <operator name="1401 examples" class="SlidingWindowValidation" expanded="yes">
       <parameter key="keep_example_set" value="true"/>
       <parameter key="training_window_width" value="400"/>
       <parameter key="test_window_width" value="350"/>
       <parameter key="horizon" value="50"/>
       <parameter key="average_performances_only" value="false"/>
       <operator name="LinearRegression" class="LinearRegression">
       </operator>
       <operator name="OperatorChain" class="OperatorChain" expanded="yes">
           <operator name="ModelApplier" class="ModelApplier">
               <parameter key="keep_model" value="true"/>
               <list key="application_parameters">
               </list>
           </operator>
           <operator name="RegressionPerformance" class="RegressionPerformance">
               <parameter key="keep_example_set" value="true"/>
               <parameter key="absolute_error" value="true"/>
               <parameter key="correlation" value="true"/>
           </operator>
           <operator name="id-0" class="Data2Log">
               <parameter key="attribute_name" value="id-0"/>
               <parameter key="example_index" value="-1"/>
           </operator>
           <operator name="sin(id/100)-0" class="Data2Log">
               <parameter key="attribute_name" value="sin(id/100)-0"/>
               <parameter key="example_index" value="-1"/>
           </operator>
           <operator name="prediction(sin(id/100)-0)" class="Data2Log">
               <parameter key="attribute_name" value="prediction(sin(id/100)-0)"/>
               <parameter key="example_index" value="-1"/>
           </operator>
           <operator name="ProcessLog" class="ProcessLog">
               <list key="log">
                 <parameter key="id-0" value="operator.id-0.value.data_value"/>
                 <parameter key="sin(id/100)-0" value="operator.sin(id/100)-0.value.data_value"/>
                 <parameter key="prediction(sin(id/100)-0)" value="operator.prediction(sin(id/100)-0).value.data_value"/>
               </list>
           </operator>
       </operator>
   </operator>
</operator>

land

Hi,
just an idea, but if you want to merge all results, the following process variation could give you an impression of another way:

<operator name="Root" class="Process" expanded="yes">
    <operator name="SalesExampleSetGenerator" class="SalesExampleSetGenerator">
        <parameter key="number_examples"	value="1500"/>
    </operator>
    <operator name="FeatureValueTypeFilter" class="FeatureValueTypeFilter">
    </operator>
    <operator name="ChangeAttributeRole" class="ChangeAttributeRole">
        <parameter key="name"	value="single_price"/>
        <parameter key="target_role"	value="label"/>
    </operator>
    <operator name="SlidingWindowValidation" class="SlidingWindowValidation" expanded="yes">
        <parameter key="training_window_width"	value="240"/>
        <parameter key="test_window_width"	value="24"/>
        <parameter key="horizon"	value="24"/>
        <parameter key="average_performances_only"	value="false"/>
        <operator name="LinearRegression" class="LinearRegression">
        </operator>
        <operator name="OperatorChain" class="OperatorChain" expanded="yes">
            <operator name="ModelApplier" class="ModelApplier">
                <parameter key="keep_model"	value="true"/>
                <list key="application_parameters">
                </list>
            </operator>
            <operator name="make transaction_id regular" class="ChangeAttributeRole">
                <parameter key="name"	value="transaction_id"/>
            </operator>
            <operator name="MergeAndStoreResults" class="OperatorChain" expanded="yes">
                <operator name="ExceptionHandling" class="ExceptionHandling" expanded="yes">
                    <operator name="IOMultiplier" class="IOMultiplier">
                        <parameter key="io_object"	value="ExampleSet"/>
                    </operator>
                    <operator name="IORetriever" class="IORetriever">
                        <parameter key="name"	value="ExampleSetStore"/>
                        <parameter key="io_object"	value="ExampleSet"/>
                    </operator>
                    <operator name="ExampleSetMerge" class="ExampleSetMerge">
                        <parameter key="merge_type"	value="first_two"/>
                    </operator>
                    <operator name="IOStorer" class="IOStorer">
                        <parameter key="name"	value="ExampleSetStore"/>
                        <parameter key="io_object"	value="ExampleSet"/>
                    </operator>
                </operator>
            </operator>
            <operator name="RegressionPerformance" class="RegressionPerformance" breakpoints="after">
                <parameter key="keep_example_set"	value="true"/>
                <parameter key="absolute_error"	value="true"/>
                <parameter key="correlation"	value="true"/>
            </operator>
        </operator>
    </operator>
</operator>

I did not really test the process, so the setup might fail. But the way should be clear...

Greetings,
  Sebastian

wessel

I think it works like this:
Although I don't understand why the 2 RegressionPerformance don't give the same results.
21 vs 22...

Regards,

Wessel

<operator name="Root" class="Process" expanded="yes">
    <operator name="SalesExampleSetGenerator" class="SalesExampleSetGenerator">
        <parameter key="number_examples" value="1500"/>
    </operator>
    <operator name="FeatureValueTypeFilter" class="FeatureValueTypeFilter">
    </operator>
    <operator name="ChangeAttributeRole" class="ChangeAttributeRole">
        <parameter key="name" value="single_price"/>
        <parameter key="target_role" value="label"/>
    </operator>
    <operator name="SlidingWindowValidation" class="SlidingWindowValidation" expanded="yes">
        <parameter key="training_window_width" value="240"/>
        <parameter key="test_window_width" value="24"/>
        <parameter key="horizon" value="24"/>
        <parameter key="average_performances_only" value="false"/>
        <operator name="LinearRegression" class="LinearRegression">
        </operator>
        <operator name="OperatorChain" class="OperatorChain" expanded="yes">
            <operator name="ModelApplier" class="ModelApplier">
                <parameter key="keep_model" value="true"/>
                <list key="application_parameters">
                </list>
            </operator>
            <operator name="make transaction_id regular" class="ChangeAttributeRole">
                <parameter key="name" value="transaction_id"/>
            </operator>
            <operator name="MergeAndStoreResults" class="OperatorChain" expanded="yes">
                <operator name="IOMultiplier" class="IOMultiplier">
                    <parameter key="io_object" value="ExampleSet"/>
                </operator>
                <operator name="ExceptionHandling" class="ExceptionHandling" expanded="yes">
                    <operator name="IORetriever (3)" class="IORetriever">
                        <parameter key="name" value="ExampleSetStore"/>
                        <parameter key="io_object" value="ExampleSet"/>
                        <parameter key="remove_from_store" value="false"/>
                    </operator>
                </operator>
                <operator name="ExampleSetMerge" class="ExampleSetMerge">
                    <parameter key="merge_type" value="first_two"/>
                </operator>
                <operator name="IOStorer" class="IOStorer">
                    <parameter key="name" value="ExampleSetStore"/>
                    <parameter key="io_object" value="ExampleSet"/>
                </operator>
            </operator>
            <operator name="IORetriever (2)" class="IORetriever">
                <parameter key="name" value="ExampleSetStore"/>
                <parameter key="io_object" value="ExampleSet"/>
                <parameter key="remove_from_store" value="false"/>
            </operator>
            <operator name="RegressionPerformance" class="RegressionPerformance">
                <parameter key="keep_example_set" value="true"/>
                <parameter key="absolute_error" value="true"/>
                <parameter key="correlation" value="true"/>
            </operator>
            <operator name="IOMultiplier (2)" class="IOMultiplier" activated="no">
                <parameter key="io_object" value="PerformanceVector"/>
            </operator>
        </operator>
    </operator>
    <operator name="IORetriever" class="IORetriever">
        <parameter key="name" value="ExampleSetStore"/>
        <parameter key="io_object" value="ExampleSet"/>
        <parameter key="remove_from_store" value="false"/>
    </operator>
    <operator name="RegressionPerformance (2)" class="RegressionPerformance">
        <parameter key="keep_example_set" value="true"/>
        <parameter key="absolute_error" value="true"/>
        <parameter key="correlation" value="true"/>
    </operator>
</operator>