"How to aggregate example sets in a CSV files"

acesari

Hello,

I am currently giving a try at rapidminer for time series prediction.
I have a setup where I make a windowed data set, and make a prediction for the next event in my time series.
I would like to save as CSV the prediction, but the CSV writer does not support appending apparently, and overwrites the file for each new prediction made. Is there a way to do this easily? I am sure there are far better processes to do something like this...

Thank you

Alexandre

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.0">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" expanded="true" name="Process">
    <process expanded="true" height="710" width="1518">
      <operator activated="true" class="read_csv" expanded="true" height="60" name="Read CSV" width="90" x="45" y="30">
        <parameter key="file_name" value="/home/cesari/eclipse/Grid/MyStrategies/marketData/EURODOLLAR_Five_Minutes.csv"/>
        <parameter key="use_quotes" value="false"/>
        <parameter key="column_separators" value=";"/>
        <parameter key="date_format" value="/MM/dd HH:mm:ss.SSS"/>
      </operator>
      <operator activated="true" class="read_csv" expanded="true" height="60" name="Read CSV (2)" width="90" x="45" y="255">
        <parameter key="file_name" value="/home/cesari/eclipse/Grid/grid-analysis/results/Positions.csv"/>
        <parameter key="use_quotes" value="false"/>
        <parameter key="column_separators" value=";"/>
        <parameter key="date_format" value="/MM/dd HH:mm:ss.SSS"/>
      </operator>
      <operator activated="true" class="discretize_by_user_specification" expanded="true" height="94" name="Discretize" width="90" x="179" y="255">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="Value"/>
        <list key="classes">
          <parameter key="short" value="-1.0"/>
          <parameter key="flat" value="0.0"/>
          <parameter key="long" value="1.0"/>
        </list>
      </operator>
      <operator activated="true" class="set_role" expanded="true" height="76" name="Set Role (2)" width="90" x="308" y="255">
        <parameter key="name" value="date"/>
        <parameter key="target_role" value="id"/>
      </operator>
      <operator activated="true" class="set_role" expanded="true" height="76" name="Set Role" width="90" x="308" y="30">
        <parameter key="name" value="Date"/>
        <parameter key="target_role" value="id"/>
      </operator>
      <operator activated="true" class="join" expanded="true" height="76" name="Join" width="90" x="442" y="30"/>
      <operator activated="true" class="set_role" expanded="true" height="76" name="Set Role (3)" width="90" x="576" y="30">
        <parameter key="name" value="Value"/>
        <parameter key="target_role" value="label"/>
      </operator>
      <operator activated="true" class="select_attributes" expanded="true" height="76" name="Select Attributes" width="90" x="718" y="32">
        <parameter key="attribute_filter_type" value="subset"/>
        <parameter key="attributes" value="date|Value|Open|Low|High|Close"/>
      </operator>
      <operator activated="true" class="multiply" expanded="true" height="76" name="Multiply" width="90" x="884" y="31"/>
      <operator activated="true" class="series:windowing" expanded="true" height="76" name="Windowing" width="90" x="1050" y="30">
        <parameter key="horizon" value="1"/>
        <parameter key="window_size" value="200"/>
        <parameter key="create_label" value="true"/>
        <parameter key="label_attribute" value="Value"/>
      </operator>
      <operator activated="true" class="series:sliding_window_validation" expanded="true" height="112" name="Validation" width="90" x="1251" y="30">
        <parameter key="training_window_width" value="50"/>
        <parameter key="test_window_width" value="1"/>
        <process expanded="true" height="673" width="419">
          <operator activated="true" class="support_vector_machine_libsvm" expanded="true" height="76" name="SVM" width="90" x="179" y="30">
            <parameter key="gamma" value="0.2"/>
            <parameter key="C" value="0.5"/>
            <parameter key="cache_size" value="200"/>
            <list key="class_weights"/>
          </operator>
          <connect from_port="training" to_op="SVM" to_port="training set"/>
          <connect from_op="SVM" from_port="model" to_port="model"/>
          <portSpacing port="source_training" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true" height="673" width="419">
          <operator activated="true" class="apply_model" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
            <list key="application_parameters"/>
          </operator>
          <operator activated="true" class="multiply" expanded="true" height="94" name="Multiply (2)" width="90" x="45" y="210"/>
          <operator activated="true" class="select_attributes" expanded="true" height="76" name="Select Attributes (2)" width="90" x="112" y="390">
            <parameter key="attribute_filter_type" value="subset"/>
            <parameter key="attribute" value="prediction(label)"/>
            <parameter key="attributes" value="prediction(label)"/>
          </operator>
          <operator activated="true" class="write_csv" expanded="true" height="60" name="Write CSV" width="90" x="255" y="399">
            <parameter key="csv_file" value="/home/cesari/Bureau/previsions.csv"/>
          </operator>
          <operator activated="true" class="performance" expanded="true" height="76" name="Performance" width="90" x="246" y="30"/>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Multiply (2)" to_port="input"/>
          <connect from_op="Multiply (2)" from_port="output 1" to_op="Performance" to_port="labelled data"/>
          <connect from_op="Multiply (2)" from_port="output 2" to_op="Select Attributes (2)" to_port="example set input"/>
          <connect from_op="Select Attributes (2)" from_port="example set output" to_op="Write CSV" to_port="input"/>
          <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_averagable 1" spacing="0"/>
          <portSpacing port="sink_averagable 2" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Read CSV" from_port="output" to_op="Set Role" to_port="example set input"/>
      <connect from_op="Read CSV (2)" from_port="output" to_op="Discretize" to_port="example set input"/>
      <connect from_op="Discretize" from_port="example set output" to_op="Set Role (2)" to_port="example set input"/>
      <connect from_op="Set Role (2)" from_port="example set output" to_op="Join" to_port="right"/>
      <connect from_op="Set Role" from_port="example set output" to_op="Join" to_port="left"/>
      <connect from_op="Join" from_port="join" to_op="Set Role (3)" to_port="example set input"/>
      <connect from_op="Set Role (3)" from_port="example set output" to_op="Select Attributes" to_port="example set input"/>
      <connect from_op="Select Attributes" from_port="example set output" to_op="Multiply" to_port="input"/>
      <connect from_op="Multiply" from_port="output 1" to_op="Windowing" to_port="example set input"/>
      <connect from_op="Windowing" from_port="example set output" to_op="Validation" to_port="training"/>
      <connect from_op="Validation" from_port="averagable 1" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>

acesari

Answer to my own question:

Easiest way for me was to use Write Special Format instead of CSV export.
Apparently many operators support appending data. It's pretty bad that the CSV doesn't.

Alexadre

land

Hi,
we are currently revising the the operators, but to be sure, you should go to bugs.rapid-i.com and insert a feature request for that, if not already present.

Greetings,
  Sebastian