Altair RISE

A program to recognize and reward our most engaged community members

Nominate Yourself Now!

Collect each performance of a Backward elimination

I am using rapid miner for my data mining research, I used backward elimination for my feature (attribute) selections. I was wondering how to set up the process in order to gather each performance for the backward elimination. For example: feature set one (A, B, C , D, E, F), performance one(…); feature set two(A, B, C, D, E), performance two(…); ….

I am currently processing a data table with 21 features and 157000 items. A brute force feature selection simply overload my computer memory. I was wonder how to find the best combination as well as plot a graph that shows which combination of features performance low, and which combination performance high.

Thanks in advance for your kindly support.

Find more posts tagged with

AI Studio

Performance

Accepted answers

All comments

wessel

This script reads the iris dataset, splits it in a test and training set and then logs performance for each subset on the test set.

I had the same problem, hope this helps.
If you need further refinement let me know.

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.0">
<context>
<input>
<location/>
</input>
<output>
<location/>
<location/>
<location/>
</output>
<macros/>
</context>
<operator activated="true" class="process" expanded="true" name="Root">
<process expanded="true" height="638" width="573">
<operator activated="true" class="retrieve" expanded="true" height="60" name="Retrieve" width="90" x="45" y="30">
<parameter key="repository_entry" value="//Samples/data/Iris"/>
</operator>
<operator activated="true" class="rename_by_replacing" expanded="true" height="76" name="rename" width="90" x="45" y="120">
<parameter key="replace_what" value="attribute_"/>
<parameter key="replace_by" value="a"/>
</operator>
<operator activated="true" class="split_data" expanded="true" height="94" name="split" width="90" x="179" y="30">
<enumeration key="partitions">
<parameter key="ratio" value="0.5"/>
<parameter key="ratio" value="0.5"/>
</enumeration>
</operator>
<operator activated="true" class="optimize_selection" expanded="true" height="94" name="FS" width="90" x="313" y="30">
<process expanded="true" height="638" width="573">
<operator activated="true" class="select_attributes" expanded="true" height="76" name="no special" width="90" x="45" y="30">
<parameter key="attribute_filter_type" value="regular_expression"/>
<parameter key="regular_expression" value="class"/>
<parameter key="invert_selection" value="true"/>
<parameter key="include_special_attributes" value="true"/>
</operator>
<operator activated="true" class="execute_script" expanded="true" height="76" name="Execute Script" width="90" x="179" y="30">
<parameter key="script" value="ExampleSet exampleSet = operator.getInput(ExampleSet.class);

String atts = "";

for (Attribute attribute : exampleSet.getAttributes()) {
 String name = attribute.getName();
	atts = atts + " " + name;
}

operator.getProcess().getMacroHandler().addMacro("atts", atts)"/>
</operator>
<operator activated="true" class="weka:W-J48" expanded="true" height="76" name="W-J48" width="90" x="179" y="165"/>
<operator activated="true" class="apply_model" expanded="true" height="76" name="Applier" width="90" x="112" y="300">
<list key="application_parameters"/>
</operator>
<operator activated="true" class="performance" expanded="true" height="76" name="Performance" width="90" x="246" y="300"/>
<operator activated="true" class="provide_macro_as_log_value" expanded="true" height="76" name="Provide Macro as Log Value" width="90" x="313" y="30">
<parameter key="macro_name" value="atts"/>
</operator>
<operator activated="true" class="log" expanded="true" height="94" name="MyLog" width="90" x="380" y="120">
<list key="log">
<parameter key="generation" value="operator.FS.value.generation"/>
<parameter key="performance" value="operator.Performance.value.performance"/>
<parameter key="atts" value="operator.Provide Macro as Log Value.value.macro_value"/>
</list>
</operator>
<connect from_port="example set" to_op="no special" to_port="example set input"/>
<connect from_port="through 1" to_op="Applier" to_port="unlabelled data"/>
<connect from_op="no special" from_port="example set output" to_op="Execute Script" to_port="input 1"/>
<connect from_op="no special" from_port="original" to_op="W-J48" to_port="training set"/>
<connect from_op="Execute Script" from_port="output 1" to_op="Provide Macro as Log Value" to_port="through 1"/>
<connect from_op="W-J48" from_port="model" to_op="Applier" to_port="model"/>
<connect from_op="Applier" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
<connect from_op="Performance" from_port="performance" to_op="MyLog" to_port="through 1"/>
<connect from_op="Provide Macro as Log Value" from_port="through 1" to_op="MyLog" to_port="through 2"/>
<connect from_op="MyLog" from_port="through 1" to_port="performance"/>
<portSpacing port="source_example set" spacing="0"/>
<portSpacing port="source_through 1" spacing="0"/>
<portSpacing port="source_through 2" spacing="0"/>
<portSpacing port="sink_performance" spacing="0"/>
</process>
</operator>
<operator activated="true" class="log_to_data" expanded="true" height="94" name="Log to Data" width="90" x="447" y="120">
<parameter key="log_name" value="MyLog"/>
</operator>
<connect from_op="Retrieve" from_port="output" to_op="rename" to_port="example set input"/>
<connect from_op="rename" from_port="example set output" to_op="split" to_port="example set"/>
<connect from_op="split" from_port="partition 1" to_op="FS" to_port="example set in"/>
<connect from_op="split" from_port="partition 2" to_op="FS" to_port="through 1"/>
<connect from_op="FS" from_port="weights" to_port="result 1"/>
<connect from_op="FS" from_port="performance" to_op="Log to Data" to_port="through 1"/>
<connect from_op="Log to Data" from_port="exampleSet" to_port="result 2"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<portSpacing port="sink_result 2" spacing="0"/>
<portSpacing port="sink_result 3" spacing="0"/>
</process>
</operator>
</process>

wessel

Is this quirky script still needed?

Or can you now log the performance of a sub process directly?
And more importantly log the "attributes used" at each iteration of attribute selection.

land

Hi,
actually this isn't needed. Was it ever? Don't know exactly...Take a look at the following process:

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.0">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="5.0.8" expanded="true" name="Process">
    <process expanded="true" height="338" width="616">
      <operator activated="true" class="retrieve" compatibility="5.0.8" expanded="true" height="60" name="Retrieve" width="90" x="45" y="30">
        <parameter key="repository_entry" value="//Samples/data/Iris"/>
      </operator>
      <operator activated="true" class="optimize_selection_forward" compatibility="5.0.10" expanded="true" height="94" name="First Round" width="90" x="179" y="30">
        <parameter key="maximal_number_of_attributes" value="25"/>
        <parameter key="speculative_rounds" value="3"/>
        <process expanded="true" height="597" width="804">
          <operator activated="true" class="materialize_data" compatibility="5.0.10" expanded="true" height="76" name="Materialize Data" width="90" x="45" y="30"/>
          <operator activated="true" class="x_validation" compatibility="5.0.10" expanded="true" height="112" name="Validation" width="90" x="179" y="30">
            <process expanded="true" height="615" width="377">
              <operator activated="true" class="normalize" compatibility="5.0.10" expanded="true" height="94" name="Normalize" width="90" x="45" y="30">
                <parameter key="create_view" value="true"/>
              </operator>
              <operator activated="true" class="linear_regression" compatibility="5.0.10" expanded="true" height="94" name="Linear Regression" width="90" x="179" y="30">
                <parameter key="feature_selection" value="none"/>
                <parameter key="eliminate_colinear_features" value="false"/>
              </operator>
              <connect from_port="training" to_op="Normalize" to_port="example set input"/>
              <connect from_op="Normalize" from_port="example set output" to_op="Linear Regression" to_port="training set"/>
              <connect from_op="Normalize" from_port="preprocessing model" to_port="through 1"/>
              <connect from_op="Linear Regression" from_port="model" to_port="model"/>
              <portSpacing port="source_training" spacing="0"/>
              <portSpacing port="sink_model" spacing="0"/>
              <portSpacing port="sink_through 1" spacing="0"/>
              <portSpacing port="sink_through 2" spacing="0"/>
            </process>
            <process expanded="true" height="615" width="413">
              <operator activated="true" class="apply_model" compatibility="5.0.10" expanded="true" height="76" name="Apply Model (2)" width="90" x="66" y="104">
                <list key="application_parameters"/>
                <parameter key="create_view" value="true"/>
              </operator>
              <operator activated="true" class="apply_model" compatibility="5.0.10" expanded="true" height="76" name="Apply Model" width="90" x="179" y="30">
                <list key="application_parameters"/>
              </operator>
              <operator activated="true" class="performance_regression" compatibility="5.0.10" expanded="true" height="76" name="Performance" width="90" x="313" y="30">
                <parameter key="absolute_error" value="true"/>
                <parameter key="squared_correlation" value="true"/>
              </operator>
              <connect from_port="model" to_op="Apply Model" to_port="model"/>
              <connect from_port="test set" to_op="Apply Model (2)" to_port="unlabelled data"/>
              <connect from_port="through 1" to_op="Apply Model (2)" to_port="model"/>
              <connect from_op="Apply Model (2)" from_port="labelled data" to_op="Apply Model" to_port="unlabelled data"/>
              <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
              <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
              <portSpacing port="source_model" spacing="0"/>
              <portSpacing port="source_test set" spacing="0"/>
              <portSpacing port="source_through 1" spacing="0"/>
              <portSpacing port="source_through 2" spacing="0"/>
              <portSpacing port="sink_averagable 1" spacing="0"/>
              <portSpacing port="sink_averagable 2" spacing="0"/>
            </process>
          </operator>
          <operator activated="true" class="log" compatibility="5.0.10" expanded="true" height="76" name="Log" width="90" x="380" y="30">
            <list key="log">
              <parameter key="numberOfAttributes" value="operator.First Round.value.number of attributes"/>
              <parameter key="attributes" value="operator.First Round.value.feature_names"/>
              <parameter key="absolute error" value="operator.Validation.value.performance2"/>
              <parameter key="std" value="operator.Validation.value.deviation"/>
              <parameter key="sqrd cor" value="operator.Validation.value.performance3"/>
            </list>
          </operator>
          <connect from_port="example set" to_op="Materialize Data" to_port="example set input"/>
          <connect from_op="Materialize Data" from_port="example set output" to_op="Validation" to_port="training"/>
          <connect from_op="Validation" from_port="averagable 1" to_op="Log" to_port="through 1"/>
          <connect from_op="Log" from_port="through 1" to_port="performance"/>
          <portSpacing port="source_example set" spacing="0"/>
          <portSpacing port="sink_performance" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Retrieve" from_port="output" to_op="First Round" to_port="example set"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
    </process>
  </operator>
</process>

Greetings,
Sebastian