Dummy-Encoding of Movie Genres

Weafer

Hello Community,

I am new to Rapidminer and currently trying to prepare a dataset through turbo-prep and I am stuck at the movie genres.

The genres are given like : ['Action', 'Drama', ...]

I would like to preserve the genres as a attribute, to do this I tried to dummy encode them but then each combination of genres gets created as a column. My goal would be a binary column for each genre.

My Question is: Is there a way to achive what I described?

BalazsBaranyRM

Hi,

the most straightforward way to do this is Generate Attributes with formulas like Genre_Action = if(contains(Genre, "'Action'"), 1, 0).
There seem to be about 20 genres, so this is doable manually. To get a list, use the Split operator on the Genre attribute and look at the results (Statistics => Genre_1 => Values). 

If the number of genres is higher, you probably prefer an automatic solution. Here's a proposed one.

First it removes unwanted characters from the genre list: ['Action', 'Adventure'] becomes Action, Adventure. Then it uses Split to split up the genre names on the commas. This gives us new columns like Genre_1, Genre_2 etc. These are appended, the values de-duplicated, this gives us a genre list.
With Loop Values we can use this list to automatically create the dummy coded columns. We use Generate Attributes for this, with the loop value as a macro (%{genre}) in the newly created attribute name and search term. 

Here's the process XML, you can paste it into the XML view in your Studio (you'll need to change the CSV file path).

<?xml version="1.0" encoding="UTF-8"?><process version="9.10.001">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="9.10.001" expanded="true" name="Process">
    <parameter key="logverbosity" value="init"/>
    <parameter key="random_seed" value="-1"/>
    <parameter key="send_mail" value="never"/>
    <parameter key="notification_email" value=""/>
    <parameter key="process_duration_for_mail" value="30"/>
    <parameter key="encoding" value="SYSTEM"/>
    <process expanded="true">
      <operator activated="true" class="read_csv" compatibility="9.10.001" expanded="true" height="68" name="Read CSV" width="90" x="45" y="34">
        <parameter key="csv_file" value="/home/barany/Downloads/MoviesCSV.csv"/>
        <parameter key="column_separators" value=";"/>
        <parameter key="trim_lines" value="false"/>
        <parameter key="use_quotes" value="true"/>
        <parameter key="quotes_character" value="&quot;"/>
        <parameter key="escape_character" value="\"/>
        <parameter key="skip_comments" value="true"/>
        <parameter key="comment_characters" value="#"/>
        <parameter key="starting_row" value="1"/>
        <parameter key="parse_numbers" value="true"/>
        <parameter key="decimal_character" value=","/>
        <parameter key="grouped_digits" value="false"/>
        <parameter key="grouping_character" value=","/>
        <parameter key="infinity_representation" value=""/>
        <parameter key="date_format" value=""/>
        <parameter key="first_row_as_names" value="true"/>
        <list key="annotations"/>
        <parameter key="time_zone" value="SYSTEM"/>
        <parameter key="locale" value="English (United States)"/>
        <parameter key="encoding" value="UTF-8"/>
        <parameter key="read_all_values_as_polynominal" value="false"/>
        <list key="data_set_meta_data_information">
          <parameter key="0" value="Column1.true.integer.attribute"/>
          <parameter key="1" value="Title.true.polynominal.attribute"/>
          <parameter key="2" value="Movie Info.true.polynominal.attribute"/>
          <parameter key="3" value="Distributor.true.polynominal.attribute"/>
          <parameter key="4" value="Release Date.true.polynominal.attribute"/>
          <parameter key="5" value="Domestic Sales (in $).true.polynominal.attribute"/>
          <parameter key="6" value="International Sales (in $).true.integer.attribute"/>
          <parameter key="7" value="World Sales (in $).true.real.attribute"/>
          <parameter key="8" value="Genre.true.polynominal.attribute"/>
          <parameter key="9" value="Movie Runtime.true.polynominal.attribute"/>
          <parameter key="10" value="License.true.polynominal.attribute"/>
        </list>
        <parameter key="read_not_matching_values_as_missings" value="false"/>
      </operator>
      <operator activated="true" class="replace" compatibility="9.10.001" expanded="true" height="82" name="Replace [] in Genre" width="90" x="179" y="34">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="Genre"/>
        <parameter key="attributes" value=""/>
        <parameter key="use_except_expression" value="false"/>
        <parameter key="value_type" value="nominal"/>
        <parameter key="use_value_type_exception" value="false"/>
        <parameter key="except_value_type" value="file_path"/>
        <parameter key="block_type" value="single_value"/>
        <parameter key="use_block_type_exception" value="false"/>
        <parameter key="except_block_type" value="single_value"/>
        <parameter key="invert_selection" value="false"/>
        <parameter key="include_special_attributes" value="false"/>
        <parameter key="replace_what" value="^\[|\]$"/>
      </operator>
      <operator activated="true" class="replace" compatibility="9.10.001" expanded="true" height="82" name="Replace ' in Genre" width="90" x="313" y="34">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="Genre"/>
        <parameter key="attributes" value=""/>
        <parameter key="use_except_expression" value="false"/>
        <parameter key="value_type" value="nominal"/>
        <parameter key="use_value_type_exception" value="false"/>
        <parameter key="except_value_type" value="file_path"/>
        <parameter key="block_type" value="single_value"/>
        <parameter key="use_block_type_exception" value="false"/>
        <parameter key="except_block_type" value="single_value"/>
        <parameter key="invert_selection" value="false"/>
        <parameter key="include_special_attributes" value="false"/>
        <parameter key="replace_what" value="'"/>
      </operator>
      <operator activated="true" class="subprocess" compatibility="9.10.001" expanded="true" height="103" name="Create attribute list" width="90" x="447" y="34">
        <process expanded="true">
          <operator activated="true" class="split" compatibility="9.10.001" expanded="true" height="82" name="Split" width="90" x="45" y="34">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Genre"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="nominal"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="file_path"/>
            <parameter key="block_type" value="single_value"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="single_value"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="split_pattern" value=","/>
            <parameter key="split_mode" value="ordered_split"/>
          </operator>
          <operator activated="true" class="multiply" compatibility="9.10.001" expanded="true" height="145" name="Multiply" width="90" x="45" y="187"/>
          <operator activated="true" class="select_attributes" compatibility="9.10.001" expanded="true" height="82" name="Genre 4" width="90" x="179" y="391">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Genre_4"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="rename_by_replacing" compatibility="9.10.001" expanded="true" height="82" name="Rename 4" width="90" x="313" y="391">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="replace_what" value="Genre.+"/>
            <parameter key="replace_by" value="GenreList"/>
          </operator>
          <operator activated="true" class="select_attributes" compatibility="9.10.001" expanded="true" height="82" name="Genre 3" width="90" x="179" y="289">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Genre_3"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="rename_by_replacing" compatibility="9.10.001" expanded="true" height="82" name="Rename 3" width="90" x="313" y="289">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="replace_what" value="Genre.+"/>
            <parameter key="replace_by" value="GenreList"/>
          </operator>
          <operator activated="true" class="select_attributes" compatibility="9.10.001" expanded="true" height="82" name="Genre 1" width="90" x="179" y="85">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Genre_1"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="rename_by_replacing" compatibility="9.10.001" expanded="true" height="82" name="Rename 1" width="90" x="313" y="85">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="replace_what" value="Genre.+"/>
            <parameter key="replace_by" value="GenreList"/>
          </operator>
          <operator activated="true" class="select_attributes" compatibility="9.10.001" expanded="true" height="82" name="Genre 2" width="90" x="179" y="187">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Genre_2"/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="rename_by_replacing" compatibility="9.10.001" expanded="true" height="82" name="Rename 2" width="90" x="313" y="187">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="replace_what" value="Genre.+"/>
            <parameter key="replace_by" value="GenreList"/>
          </operator>
          <operator activated="true" class="append" compatibility="9.10.001" expanded="true" height="145" name="Append" width="90" x="447" y="238">
            <parameter key="datamanagement" value="double_array"/>
            <parameter key="data_management" value="auto"/>
            <parameter key="merge_type" value="all"/>
          </operator>
          <operator activated="true" class="trim" compatibility="9.10.001" expanded="true" height="82" name="Trim" width="90" x="581" y="238">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="nominal"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="file_path"/>
            <parameter key="block_type" value="single_value"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="single_value"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
          </operator>
          <operator activated="true" class="filter_examples" compatibility="9.10.001" expanded="true" height="103" name="Filter Examples" width="90" x="715" y="238">
            <parameter key="parameter_expression" value=""/>
            <parameter key="condition_class" value="custom_filters"/>
            <parameter key="invert_filter" value="false"/>
            <list key="filters_list">
              <parameter key="filters_entry_key" value="GenreList.is_not_missing."/>
            </list>
            <parameter key="filters_logic_and" value="true"/>
            <parameter key="filters_check_metadata" value="true"/>
          </operator>
          <operator activated="true" class="remove_duplicates" compatibility="9.10.001" expanded="true" height="103" name="Remove Duplicates" width="90" x="849" y="136">
            <parameter key="attribute_filter_type" value="all"/>
            <parameter key="attribute" value=""/>
            <parameter key="attributes" value=""/>
            <parameter key="use_except_expression" value="false"/>
            <parameter key="value_type" value="attribute_value"/>
            <parameter key="use_value_type_exception" value="false"/>
            <parameter key="except_value_type" value="time"/>
            <parameter key="block_type" value="attribute_block"/>
            <parameter key="use_block_type_exception" value="false"/>
            <parameter key="except_block_type" value="value_matrix_row_start"/>
            <parameter key="invert_selection" value="false"/>
            <parameter key="include_special_attributes" value="false"/>
            <parameter key="treat_missing_values_as_duplicates" value="false"/>
          </operator>
          <connect from_port="in 1" to_op="Split" to_port="example set input"/>
          <connect from_op="Split" from_port="example set output" to_op="Multiply" to_port="input"/>
          <connect from_op="Split" from_port="original" to_port="out 2"/>
          <connect from_op="Multiply" from_port="output 1" to_op="Genre 1" to_port="example set input"/>
          <connect from_op="Multiply" from_port="output 2" to_op="Genre 2" to_port="example set input"/>
          <connect from_op="Multiply" from_port="output 3" to_op="Genre 3" to_port="example set input"/>
          <connect from_op="Multiply" from_port="output 4" to_op="Genre 4" to_port="example set input"/>
          <connect from_op="Genre 4" from_port="example set output" to_op="Rename 4" to_port="example set input"/>
          <connect from_op="Rename 4" from_port="example set output" to_op="Append" to_port="example set 4"/>
          <connect from_op="Genre 3" from_port="example set output" to_op="Rename 3" to_port="example set input"/>
          <connect from_op="Rename 3" from_port="example set output" to_op="Append" to_port="example set 3"/>
          <connect from_op="Genre 1" from_port="example set output" to_op="Rename 1" to_port="example set input"/>
          <connect from_op="Rename 1" from_port="example set output" to_op="Append" to_port="example set 1"/>
          <connect from_op="Genre 2" from_port="example set output" to_op="Rename 2" to_port="example set input"/>
          <connect from_op="Rename 2" from_port="example set output" to_op="Append" to_port="example set 2"/>
          <connect from_op="Append" from_port="merged set" to_op="Trim" to_port="example set input"/>
          <connect from_op="Trim" from_port="example set output" to_op="Filter Examples" to_port="example set input"/>
          <connect from_op="Filter Examples" from_port="example set output" to_op="Remove Duplicates" to_port="example set input"/>
          <connect from_op="Remove Duplicates" from_port="example set output" to_port="out 1"/>
          <portSpacing port="source_in 1" spacing="0"/>
          <portSpacing port="source_in 2" spacing="0"/>
          <portSpacing port="sink_out 1" spacing="0"/>
          <portSpacing port="sink_out 2" spacing="0"/>
          <portSpacing port="sink_out 3" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="concurrency:loop_values" compatibility="9.10.001" expanded="true" height="103" name="Loop genres" width="90" x="581" y="34">
        <parameter key="attribute" value="GenreList"/>
        <parameter key="iteration_macro" value="genre"/>
        <parameter key="reuse_results" value="true"/>
        <parameter key="enable_parallel_execution" value="true"/>
        <process expanded="true">
          <operator activated="true" class="generate_attributes" compatibility="9.10.001" expanded="true" height="82" name="Generate Attributes" width="90" x="112" y="85">
            <list key="function_descriptions">
              <parameter key="Genre_%{genre}" value="if(contains(Genre, %{genre}), 1, 0)"/>
            </list>
            <parameter key="keep_all" value="true"/>
          </operator>
          <connect from_port="input 1" to_port="output 1"/>
          <connect from_port="input 2" to_op="Generate Attributes" to_port="example set input"/>
          <connect from_op="Generate Attributes" from_port="example set output" to_port="output 2"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="source_input 2" spacing="0"/>
          <portSpacing port="source_input 3" spacing="0"/>
          <portSpacing port="sink_output 1" spacing="0"/>
          <portSpacing port="sink_output 2" spacing="0"/>
          <portSpacing port="sink_output 3" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Read CSV" from_port="output" to_op="Replace [] in Genre" to_port="example set input"/>
      <connect from_op="Replace [] in Genre" from_port="example set output" to_op="Replace ' in Genre" to_port="example set input"/>
      <connect from_op="Replace ' in Genre" from_port="example set output" to_op="Create attribute list" to_port="in 1"/>
      <connect from_op="Create attribute list" from_port="out 1" to_op="Loop genres" to_port="input 1"/>
      <connect from_op="Create attribute list" from_port="out 2" to_op="Loop genres" to_port="input 2"/>
      <connect from_op="Loop genres" from_port="output 2" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>

Regards,
Balázs