How to calculate Micro Recall and Precision, and Macro Recall and Precision in RapidMiner.

Sajin

How to calculate Micro Recall and Precision, and Macro Recall and precision in RapidMiner. Do we have any operator for the same. My problem statement goes like this....

I am getting n*n (say 3*3, 4*4, 5*5 and so on.... Till 10*10 matrix) confusion matrix from the ''Multi Label Performance Vector". I have got Precision and Recall for the Individual label's. I want to calculate Micro Recall and Precision, and Macro Recall and Precision. Do we have any operator for the same.? 

Quicker responses are much appreciated. 

Below is the sample of my process

<?xml version="1.0" encoding="UTF-8"?><process version="9.9.000">

  <context>

    <input/>

    <output/>

    <macros/>

  </context>

  <operator activated="true" class="process" compatibility="9.9.000" expanded="true" name="Process">

    <parameter key="logverbosity" value="init"/>

    <parameter key="random_seed" value="2001"/>

    <parameter key="send_mail" value="never"/>

    <parameter key="notification_email" value=""/>

    <parameter key="process_duration_for_mail" value="30"/>

    <parameter key="encoding" value="SYSTEM"/>

    <process expanded="true">

      <operator activated="false" class="generate_data" compatibility="9.9.000" expanded="true" height="68" name="Generate Data" width="90" x="45" y="136">

        <parameter key="target_function" value="multi classification"/>

        <parameter key="number_examples" value="100"/>

        <parameter key="number_of_attributes" value="5"/>

        <parameter key="attributes_lower_bound" value="-10.0"/>

        <parameter key="attributes_upper_bound" value="10.0"/>

        <parameter key="gaussian_standard_deviation" value="10.0"/>

        <parameter key="largest_radius" value="10.0"/>

        <parameter key="use_local_random_seed" value="false"/>

        <parameter key="local_random_seed" value="1992"/>

        <parameter key="datamanagement" value="double_array"/>

        <parameter key="data_management" value="auto"/>

      </operator>

      <operator activated="true" class="retrieve" compatibility="9.9.000" expanded="true" height="68" name="Retrieve Iris" width="90" x="45" y="34">

        <parameter key="repository_entry" value="//Samples/data/Iris"/>

      </operator>

      <operator activated="true" class="subprocess" compatibility="9.9.000" expanded="true" height="82" name="Turbo Prep Process" width="90" x="179" y="34">

        <process expanded="true">

          <operator activated="true" class="concurrency:loop_attributes" compatibility="9.9.000" expanded="true" height="82" name="Loop Attributes" width="90" x="45" y="34">

            <parameter key="attribute_filter_type" value="regular_expression"/>

            <parameter key="attribute" value=""/>

            <parameter key="attributes" value=""/>

            <parameter key="regular_expression" value="\Qlabel\E"/>

            <parameter key="use_except_expression" value="false"/>

            <parameter key="value_type" value="attribute_value"/>

            <parameter key="use_value_type_exception" value="false"/>

            <parameter key="except_value_type" value="time"/>

            <parameter key="block_type" value="attribute_block"/>

            <parameter key="use_block_type_exception" value="false"/>

            <parameter key="except_block_type" value="value_matrix_row_start"/>

            <parameter key="invert_selection" value="false"/>

            <parameter key="include_special_attributes" value="true"/>

            <parameter key="attribute_name_macro" value="loop_attribute"/>

            <parameter key="reuse_results" value="true"/>

            <parameter key="enable_parallel_execution" value="true"/>

            <process expanded="true">

              <operator activated="true" class="set_role" compatibility="9.9.000" expanded="true" height="82" name="Set Role" width="90" x="45" y="34">

                <parameter key="attribute_name" value="%{loop_attribute}"/>

                <parameter key="target_role" value="regular"/>

                <list key="set_additional_roles"/>

              </operator>

              <connect from_port="input 1" to_op="Set Role" to_port="example set input"/>

              <connect from_op="Set Role" from_port="example set output" to_port="output 1"/>

              <portSpacing port="source_input 1" spacing="0"/>

              <portSpacing port="source_input 2" spacing="0"/>

              <portSpacing port="sink_output 1" spacing="0"/>

              <portSpacing port="sink_output 2" spacing="0"/>

            </process>

            <description align="center" color="transparent" colored="false" width="126">Change role to 'regular' for label</description>

          </operator>

          <operator activated="true" class="subprocess" compatibility="9.9.000" expanded="true" height="82" name="Subprocess" width="90" x="179" y="34">

            <process expanded="true">

              <operator activated="true" class="nominal_to_text" compatibility="9.9.000" expanded="true" height="82" name="Nominal to Text" width="90" x="45" y="34">

                <parameter key="attribute_filter_type" value="value_type"/>

                <parameter key="attribute" value=""/>

                <parameter key="attributes" value=""/>

                <parameter key="use_except_expression" value="false"/>

                <parameter key="value_type" value="nominal"/>

                <parameter key="use_value_type_exception" value="false"/>

                <parameter key="except_value_type" value="file_path"/>

                <parameter key="block_type" value="single_value"/>

                <parameter key="use_block_type_exception" value="false"/>

                <parameter key="except_block_type" value="single_value"/>

                <parameter key="invert_selection" value="false"/>

                <parameter key="include_special_attributes" value="false"/>

                <description align="center" color="transparent" colored="false" width="126">Change all categorical columns to text</description>

              </operator>

              <operator activated="true" class="text_to_nominal" compatibility="9.9.000" expanded="true" height="82" name="Text to Nominal" width="90" x="179" y="34">

                <parameter key="attribute_filter_type" value="value_type"/>

                <parameter key="attribute" value=""/>

                <parameter key="attributes" value=""/>

                <parameter key="use_except_expression" value="false"/>

                <parameter key="value_type" value="text"/>

                <parameter key="use_value_type_exception" value="false"/>

                <parameter key="except_value_type" value="text"/>

                <parameter key="block_type" value="value_matrix"/>

                <parameter key="use_block_type_exception" value="false"/>

                <parameter key="except_block_type" value="value_matrix_row_start"/>

                <parameter key="invert_selection" value="false"/>

                <parameter key="include_special_attributes" value="false"/>

                <description align="center" color="transparent" colored="false" width="126">Change all text columns to nominal</description>

              </operator>

              <operator activated="true" class="numerical_to_real" compatibility="9.9.000" expanded="true" height="82" name="Numerical to Real" width="90" x="313" y="34">

                <parameter key="attribute_filter_type" value="value_type"/>

                <parameter key="attribute" value=""/>

                <parameter key="attributes" value=""/>

                <parameter key="use_except_expression" value="false"/>

                <parameter key="value_type" value="numeric"/>

                <parameter key="use_value_type_exception" value="false"/>

                <parameter key="except_value_type" value="real"/>

                <parameter key="block_type" value="value_series"/>

                <parameter key="use_block_type_exception" value="false"/>

                <parameter key="except_block_type" value="value_series_end"/>

                <parameter key="invert_selection" value="false"/>

                <parameter key="include_special_attributes" value="false"/>

                <description align="center" color="transparent" colored="false" width="126">Change all numerical columns to real</description>

              </operator>

              <connect from_port="in 1" to_op="Nominal to Text" to_port="example set input"/>

              <connect from_op="Nominal to Text" from_port="example set output" to_op="Text to Nominal" to_port="example set input"/>

              <connect from_op="Text to Nominal" from_port="example set output" to_op="Numerical to Real" to_port="example set input"/>

              <connect from_op="Numerical to Real" from_port="example set output" to_port="out 1"/>

              <portSpacing port="source_in 1" spacing="0"/>

              <portSpacing port="source_in 2" spacing="0"/>

              <portSpacing port="sink_out 1" spacing="0"/>

              <portSpacing port="sink_out 2" spacing="0"/>

            </process>

            <description align="center" color="transparent" colored="false" width="126">Unify column types</description>

          </operator>

          <operator activated="true" class="rename" compatibility="9.9.000" expanded="true" height="82" name="Rename (Deprecated)" origin="GENERATED_TURBOPREP" width="90" x="404" y="34">

            <parameter key="old_name" value="label"/>

            <parameter key="new_name" value="newcol"/>

            <list key="rename_additional_attributes"/>

            <description align="center" color="transparent" colored="false" width="126">label -&gt; newcol</description>

          </operator>

          <operator activated="true" class="order_attributes" compatibility="9.9.000" expanded="true" height="82" name="Reorder Attributes" width="90" x="629" y="34">

            <parameter key="sort_mode" value="user specified"/>

            <parameter key="attribute_ordering" value="att1|att2|att3|att4|att5|newcol"/>

            <parameter key="use_regular_expressions" value="false"/>

            <parameter key="handle_unmatched" value="append"/>

            <parameter key="sort_direction" value="ascending"/>

            <description align="center" color="transparent" colored="false" width="126">Ensure order of columns matches the table view.</description>

          </operator>

          <connect from_port="in 1" to_op="Loop Attributes" to_port="input 1"/>

          <connect from_op="Loop Attributes" from_port="output 1" to_op="Subprocess" to_port="in 1"/>

          <connect from_op="Subprocess" from_port="out 1" to_op="Rename (Deprecated)" to_port="example set input"/>

          <connect from_op="Rename (Deprecated)" from_port="example set output" to_op="Reorder Attributes" to_port="example set input"/>

          <connect from_op="Reorder Attributes" from_port="example set output" to_port="out 1"/>

          <portSpacing port="source_in 1" spacing="0"/>

          <portSpacing port="source_in 2" spacing="0"/>

          <portSpacing port="sink_out 1" spacing="0"/>

          <portSpacing port="sink_out 2" spacing="0"/>

        </process>

        <description align="center" color="transparent" colored="false" width="126">TP Operations: Rename</description>

      </operator>

      <operator activated="true" class="set_role" compatibility="9.9.000" expanded="true" height="82" name="Set Role (2)" width="90" x="313" y="34">

        <parameter key="attribute_name" value="newcol"/>

        <parameter key="target_role" value="regular"/>

        <list key="set_additional_roles"/>

      </operator>

      <operator activated="true" class="split_data" compatibility="9.9.000" expanded="true" height="103" name="Split Data" width="90" x="447" y="85">

        <enumeration key="partitions">

          <parameter key="ratio" value="0.7"/>

          <parameter key="ratio" value="0.3"/>

        </enumeration>

        <parameter key="sampling_type" value="automatic"/>

        <parameter key="use_local_random_seed" value="false"/>

        <parameter key="local_random_seed" value="1992"/>

      </operator>

      <operator activated="true" class="time_series:multi_label_model_learner" compatibility="9.9.000" expanded="true" height="82" name="Multi Label Modeling" width="90" x="581" y="34">

        <parameter key="attribute_filter_type" value="single"/>

        <parameter key="attribute" value="newcol"/>

        <parameter key="attributes" value=""/>

        <parameter key="use_except_expression" value="false"/>

        <parameter key="value_type" value="attribute_value"/>

        <parameter key="use_value_type_exception" value="false"/>

        <parameter key="except_value_type" value="time"/>

        <parameter key="block_type" value="attribute_block"/>

        <parameter key="use_block_type_exception" value="false"/>

        <parameter key="except_block_type" value="value_matrix_row_start"/>

        <parameter key="invert_selection" value="false"/>

        <parameter key="include_special_attributes" value="true"/>

        <parameter key="add_macros" value="false"/>

        <parameter key="current_label_name_macro" value="current_label_attribute"/>

        <parameter key="current_label_type_macro" value="current_label_type"/>

        <parameter key="enable_parallel_execution" value="true"/>

        <process expanded="true">

          <operator activated="false" class="support_vector_machine_libsvm" compatibility="9.9.000" expanded="true" height="82" name="SVM" width="90" x="246" y="136">

            <parameter key="svm_type" value="C-SVC"/>

            <parameter key="kernel_type" value="rbf"/>

            <parameter key="degree" value="3"/>

            <parameter key="gamma" value="0.0"/>

            <parameter key="coef0" value="0.0"/>

            <parameter key="C" value="0.0"/>

            <parameter key="nu" value="0.5"/>

            <parameter key="cache_size" value="80"/>

            <parameter key="epsilon" value="0.001"/>

            <parameter key="p" value="0.1"/>

            <list key="class_weights"/>

            <parameter key="shrinking" value="true"/>

            <parameter key="calculate_confidences" value="false"/>

            <parameter key="confidence_for_multiclass" value="true"/>

          </operator>

          <operator activated="true" class="k_nn" compatibility="9.9.000" expanded="true" height="82" name="k-NN" width="90" x="112" y="34">

            <parameter key="k" value="5"/>

            <parameter key="weighted_vote" value="true"/>

            <parameter key="measure_types" value="MixedMeasures"/>

            <parameter key="mixed_measure" value="MixedEuclideanDistance"/>

            <parameter key="nominal_measure" value="NominalDistance"/>

            <parameter key="numerical_measure" value="EuclideanDistance"/>

            <parameter key="divergence" value="GeneralizedIDivergence"/>

            <parameter key="kernel_type" value="radial"/>

            <parameter key="kernel_gamma" value="1.0"/>

            <parameter key="kernel_sigma1" value="1.0"/>

            <parameter key="kernel_sigma2" value="0.0"/>

            <parameter key="kernel_sigma3" value="2.0"/>

            <parameter key="kernel_degree" value="3.0"/>

            <parameter key="kernel_shift" value="1.0"/>

            <parameter key="kernel_a" value="1.0"/>

            <parameter key="kernel_b" value="0.0"/>

          </operator>

          <connect from_port="training set" to_op="k-NN" to_port="training set"/>

          <connect from_op="k-NN" from_port="model" to_port="model"/>

          <portSpacing port="source_training set" spacing="0"/>

          <portSpacing port="source_input 1" spacing="0"/>

          <portSpacing port="sink_model" spacing="0"/>

          <portSpacing port="sink_output 1" spacing="0"/>

        </process>

      </operator>

      <operator activated="true" class="apply_model" compatibility="9.9.000" expanded="true" height="82" name="Apply Model" width="90" x="648" y="136">

        <list key="application_parameters"/>

        <parameter key="create_view" value="false"/>

      </operator>

      <operator activated="true" class="time_series:multi_label_performance_evaluator" compatibility="9.9.000" expanded="true" height="124" name="Multi Label Performance" width="90" x="782" y="136">

        <parameter key="auto_detect_label_and_prediction_attributes" value="true"/>

        <parameter key="attribute_filter_type" value="single"/>

        <parameter key="attribute" value="prediction(newcol)"/>

        <parameter key="attributes" value=""/>

        <parameter key="use_except_expression" value="false"/>

        <parameter key="value_type" value="attribute_value"/>

        <parameter key="use_value_type_exception" value="false"/>

        <parameter key="except_value_type" value="time"/>

        <parameter key="block_type" value="attribute_block"/>

        <parameter key="use_block_type_exception" value="false"/>

        <parameter key="except_block_type" value="value_matrix_row_start"/>

        <parameter key="invert_selection" value="false"/>

        <parameter key="include_special_attributes" value="true"/>

        <parameter key="add_macros" value="false"/>

        <parameter key="current_label_name_macro" value="current_label_attribute"/>

        <parameter key="current_label_type_macro" value="current_label_type"/>

        <parameter key="enable_parallel_execution" value="true"/>

        <process expanded="true">

          <operator activated="true" class="performance_classification" compatibility="9.9.000" expanded="true" height="82" name="Performance" width="90" x="179" y="34">

            <parameter key="main_criterion" value="first"/>

            <parameter key="accuracy" value="true"/>

            <parameter key="classification_error" value="false"/>

            <parameter key="kappa" value="true"/>

            <parameter key="weighted_mean_recall" value="true"/>

            <parameter key="weighted_mean_precision" value="true"/>

            <parameter key="spearman_rho" value="true"/>

            <parameter key="kendall_tau" value="true"/>

            <parameter key="absolute_error" value="true"/>

            <parameter key="relative_error" value="true"/>

            <parameter key="relative_error_lenient" value="true"/>

            <parameter key="relative_error_strict" value="true"/>

            <parameter key="normalized_absolute_error" value="true"/>

            <parameter key="root_mean_squared_error" value="true"/>

            <parameter key="root_relative_squared_error" value="true"/>

            <parameter key="squared_error" value="true"/>

            <parameter key="correlation" value="true"/>

            <parameter key="squared_correlation" value="true"/>

            <parameter key="cross-entropy" value="true"/>

            <parameter key="margin" value="true"/>

            <parameter key="soft_margin_loss" value="true"/>

            <parameter key="logistic_loss" value="false"/>

            <parameter key="skip_undefined_labels" value="true"/>

            <parameter key="use_example_weights" value="true"/>

            <list key="class_weights"/>

          </operator>

          <connect from_port="labelled set" to_op="Performance" to_port="labelled data"/>

          <connect from_op="Performance" from_port="performance" to_port="performance"/>

          <portSpacing port="source_labelled set" spacing="0"/>

          <portSpacing port="source_input 1" spacing="0"/>

          <portSpacing port="sink_performance" spacing="0"/>

          <portSpacing port="sink_output 1" spacing="0"/>

        </process>

      </operator>

      <operator activated="false" class="converters:confusionmatrix_2_example_set" compatibility="0.9.000" expanded="true" height="82" name="Confusion Matrix to ExampleSet" width="90" x="849" y="340"/>

      <operator activated="false" class="converters:confusionmatrix_2_example_set" compatibility="0.9.000" expanded="true" height="82" name="Confusion Matrix to ExampleSet (2)" width="90" x="514" y="238"/>

      <connect from_op="Retrieve Iris" from_port="output" to_op="Turbo Prep Process" to_port="in 1"/>

      <connect from_op="Turbo Prep Process" from_port="out 1" to_op="Set Role (2)" to_port="example set input"/>

      <connect from_op="Set Role (2)" from_port="example set output" to_op="Split Data" to_port="example set"/>

      <connect from_op="Split Data" from_port="partition 1" to_op="Multi Label Modeling" to_port="training set"/>

      <connect from_op="Split Data" from_port="partition 2" to_op="Apply Model" to_port="unlabelled data"/>

      <connect from_op="Multi Label Modeling" from_port="model" to_op="Apply Model" to_port="model"/>

      <connect from_op="Apply Model" from_port="labelled data" to_op="Multi Label Performance" to_port="labelled set"/>

      <connect from_op="Multi Label Performance" from_port="performance" to_port="result 1"/>

      <portSpacing port="source_input 1" spacing="0"/>

      <portSpacing port="sink_result 1" spacing="0"/>

      <portSpacing port="sink_result 2" spacing="0"/>

    </process>

  </operator>

</process>

MartinLiebig

Hi,

I am not exactly sure what you search for. But if it comes to the wire, you can always use Confusion Matrix to Example Set and calculate things manually?

Best,

Martin