Stem (dictionary) for greek language

Hello to the community of rapidminer,

i'm trying to create a stemmer for greek language but i can't implement a more general rule for removing punctuations. For example i want words like "fishes","fished","fishing","fishery" to be reduced to "fish". Due to the wide range of punctuations in greek language is too dificult to map every possible punctuation with the origin of the word. So i tried a rule like this:

fish:fish.*

but it didn't work out. Is there any way to do that ?

thank you in advance

Find more posts tagged with

AI Studio

Text Mining + NLP

Accepted answers

All comments

Thomas_Ott

That should work, can you post your process?

slimik

<?xml version="1.0" encoding="UTF-8"?><process version="7.3.001">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="6.0.002" expanded="true" name="Process">
    <parameter key="logverbosity" value="init"/>
    <parameter key="random_seed" value="2001"/>
    <parameter key="send_mail" value="never"/>
    <parameter key="notification_email" value=""/>
    <parameter key="process_duration_for_mail" value="30"/>
    <parameter key="encoding" value="SYSTEM"/>
    <process expanded="true">
      <operator activated="true" class="read_excel" compatibility="7.3.001" expanded="true" height="68" name="Read Excel" width="90" x="45" y="34">
        <parameter key="sheet_number" value="1"/>
        <parameter key="imported_cell_range" value="A1:C1574"/>
        <parameter key="encoding" value="SYSTEM"/>
        <parameter key="first_row_as_names" value="false"/>
        <list key="annotations">
          <parameter key="0" value="Name"/>
        </list>
        <parameter key="date_format" value=""/>
        <parameter key="time_zone" value="SYSTEM"/>
        <parameter key="locale" value="English (United States)"/>
        <list key="data_set_meta_data_information">
          <parameter key="0" value="ID.true.numeric.id"/>
          <parameter key="1" value="Tweet.true.text.attribute"/>
          <parameter key="2" value="Sentiment.true.binominal.label"/>
        </list>
        <parameter key="read_not_matching_values_as_missings" value="true"/>
        <parameter key="datamanagement" value="double_array"/>
      </operator>
      <operator activated="true" class="text:process_document_from_data" compatibility="7.3.000" expanded="true" height="82" name="Process Documents from Data" width="90" x="246" y="34">
        <parameter key="create_word_vector" value="true"/>
        <parameter key="vector_creation" value="TF-IDF"/>
        <parameter key="add_meta_information" value="true"/>
        <parameter key="keep_text" value="true"/>
        <parameter key="prune_method" value="percentual"/>
        <parameter key="prune_below_percent" value="1.0"/>
        <parameter key="prune_above_percent" value="62.4"/>
        <parameter key="prune_below_absolute" value="1"/>
        <parameter key="prune_above_absolute" value="10"/>
        <parameter key="prune_below_rank" value="0.05"/>
        <parameter key="prune_above_rank" value="0.95"/>
        <parameter key="datamanagement" value="double_sparse_array"/>
        <parameter key="select_attributes_and_weights" value="false"/>
        <list key="specify_weights"/>
        <process expanded="true">
          <operator activated="true" class="text:tokenize" compatibility="7.3.000" expanded="true" height="68" name="Tokenize" width="90" x="112" y="34">
            <parameter key="mode" value="non letters"/>
            <parameter key="characters" value=".:"/>
            <parameter key="language" value="English"/>
            <parameter key="max_token_length" value="3"/>
          </operator>
          <operator activated="true" class="text:transform_cases" compatibility="7.3.000" expanded="true" height="68" name="Transform Cases" width="90" x="246" y="34">
            <parameter key="transform_to" value="lower case"/>
          </operator>
          <operator activated="true" class="text:filter_by_length" compatibility="7.3.000" expanded="true" height="68" name="Filter Tokens (by Length)" width="90" x="112" y="238">
            <parameter key="min_chars" value="4"/>
            <parameter key="max_chars" value="30"/>
          </operator>
          <operator activated="true" class="text:stem_porter" compatibility="7.3.000" expanded="true" height="68" name="Stem (Porter)" width="90" x="246" y="187"/>
          <operator activated="true" class="text:stem_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Stem (Dictionary)" width="90" x="380" y="238">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\stemmer.txt"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Filter Stopwords (Dictionary)" width="90" x="514" y="238">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\gr_stopwords.txt"/>
            <parameter key="case_sensitive" value="false"/>
            <parameter key="encoding" value="SYSTEM"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_english" compatibility="7.3.000" expanded="true" height="68" name="Filter Stopwords (English)" width="90" x="514" y="34"/>
          <operator activated="true" class="text:generate_n_grams_terms" compatibility="7.3.000" expanded="true" height="68" name="Generate n-Grams (Terms)" width="90" x="648" y="34">
            <parameter key="max_length" value="2"/>
          </operator>
          <connect from_port="document" to_op="Tokenize" to_port="document"/>
          <connect from_op="Tokenize" from_port="document" to_op="Transform Cases" to_port="document"/>
          <connect from_op="Transform Cases" from_port="document" to_op="Filter Tokens (by Length)" to_port="document"/>
          <connect from_op="Filter Tokens (by Length)" from_port="document" to_op="Stem (Porter)" to_port="document"/>
          <connect from_op="Stem (Porter)" from_port="document" to_op="Stem (Dictionary)" to_port="document"/>
          <connect from_op="Stem (Dictionary)" from_port="document" to_op="Filter Stopwords (Dictionary)" to_port="document"/>
          <connect from_op="Filter Stopwords (Dictionary)" from_port="document" to_op="Filter Stopwords (English)" to_port="document"/>
          <connect from_op="Filter Stopwords (English)" from_port="document" to_op="Generate n-Grams (Terms)" to_port="document"/>
          <connect from_op="Generate n-Grams (Terms)" from_port="document" to_port="document 1"/>
          <portSpacing port="source_document" spacing="0"/>
          <portSpacing port="sink_document 1" spacing="0"/>
          <portSpacing port="sink_document 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="set_role" compatibility="7.3.001" expanded="true" height="82" name="Set Role" width="90" x="380" y="34">
        <parameter key="attribute_name" value="Sentiment"/>
        <parameter key="target_role" value="label"/>
        <list key="set_additional_roles"/>
      </operator>
      <operator activated="true" class="x_validation" compatibility="7.3.001" expanded="true" height="124" name="Validation" width="90" x="514" y="34">
        <parameter key="create_complete_model" value="false"/>
        <parameter key="average_performances_only" value="true"/>
        <parameter key="leave_one_out" value="false"/>
        <parameter key="number_of_validations" value="10"/>
        <parameter key="sampling_type" value="automatic"/>
        <parameter key="use_local_random_seed" value="false"/>
        <parameter key="local_random_seed" value="1992"/>
        <process expanded="true">
          <operator activated="true" class="support_vector_machine" compatibility="7.3.001" expanded="true" height="124" name="SVM (2)" width="90" x="179" y="34">
            <parameter key="kernel_type" value="radial"/>
            <parameter key="kernel_gamma" value="0.0012760559257285525"/>
            <parameter key="kernel_sigma1" value="1.0"/>
            <parameter key="kernel_sigma2" value="0.0"/>
            <parameter key="kernel_sigma3" value="2.0"/>
            <parameter key="kernel_shift" value="1.0"/>
            <parameter key="kernel_degree" value="2.0"/>
            <parameter key="kernel_a" value="1.0"/>
            <parameter key="kernel_b" value="0.0"/>
            <parameter key="kernel_cache" value="200"/>
            <parameter key="C" value="63.095734448019364"/>
            <parameter key="convergence_epsilon" value="0.001"/>
            <parameter key="max_iterations" value="100000"/>
            <parameter key="scale" value="true"/>
            <parameter key="calculate_weights" value="true"/>
            <parameter key="return_optimization_performance" value="true"/>
            <parameter key="L_pos" value="1.0"/>
            <parameter key="L_neg" value="1.0"/>
            <parameter key="epsilon" value="0.0"/>
            <parameter key="epsilon_plus" value="0.0"/>
            <parameter key="epsilon_minus" value="0.0"/>
            <parameter key="balance_cost" value="false"/>
            <parameter key="quadratic_loss_pos" value="false"/>
            <parameter key="quadratic_loss_neg" value="false"/>
            <parameter key="estimate_performance" value="false"/>
          </operator>
          <connect from_port="training" to_op="SVM (2)" to_port="training set"/>
          <connect from_op="SVM (2)" from_port="model" to_port="model"/>
          <portSpacing port="source_training" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true">
          <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model" width="90" x="45" y="34">
            <list key="application_parameters"/>
            <parameter key="create_view" value="false"/>
          </operator>
          <operator activated="true" class="performance" compatibility="7.3.001" expanded="true" height="82" name="Performance (2)" width="90" x="246" y="34">
            <parameter key="use_example_weights" value="true"/>
          </operator>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Performance (2)" to_port="labelled data"/>
          <connect from_op="Performance (2)" from_port="performance" to_port="averagable 1"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_averagable 1" spacing="0"/>
          <portSpacing port="sink_averagable 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="read_excel" compatibility="7.3.001" expanded="true" height="68" name="Read Excel (2)" width="90" x="112" y="187">
        <parameter key="sheet_number" value="1"/>
        <parameter key="imported_cell_range" value="A1:C8990"/>
        <parameter key="encoding" value="SYSTEM"/>
        <parameter key="first_row_as_names" value="false"/>
        <list key="annotations">
          <parameter key="0" value="Name"/>
        </list>
        <parameter key="date_format" value=""/>
        <parameter key="time_zone" value="SYSTEM"/>
        <parameter key="locale" value="English (United States)"/>
        <list key="data_set_meta_data_information">
          <parameter key="0" value="ID.true.numeric.id"/>
          <parameter key="1" value="Tweet.true.text.attribute"/>
          <parameter key="2" value="Sentiment.true.attribute_value.attribute"/>
        </list>
        <parameter key="read_not_matching_values_as_missings" value="true"/>
        <parameter key="datamanagement" value="double_array"/>
      </operator>
      <operator activated="true" class="text:process_document_from_data" compatibility="7.3.000" expanded="true" height="82" name="Process Apply Documents" width="90" x="313" y="187">
        <parameter key="create_word_vector" value="true"/>
        <parameter key="vector_creation" value="TF-IDF"/>
        <parameter key="add_meta_information" value="true"/>
        <parameter key="keep_text" value="true"/>
        <parameter key="prune_method" value="percentual"/>
        <parameter key="prune_below_percent" value="1.0"/>
        <parameter key="prune_above_percent" value="62.4"/>
        <parameter key="prune_below_absolute" value="1"/>
        <parameter key="prune_above_absolute" value="10"/>
        <parameter key="prune_below_rank" value="0.05"/>
        <parameter key="prune_above_rank" value="0.95"/>
        <parameter key="datamanagement" value="double_sparse_array"/>
        <parameter key="select_attributes_and_weights" value="false"/>
        <list key="specify_weights"/>
        <process expanded="true">
          <operator activated="true" class="text:tokenize" compatibility="7.3.000" expanded="true" height="68" name="Tokenize (2)" width="90" x="112" y="34">
            <parameter key="mode" value="non letters"/>
            <parameter key="characters" value=".:"/>
            <parameter key="language" value="English"/>
            <parameter key="max_token_length" value="3"/>
          </operator>
          <operator activated="true" class="text:transform_cases" compatibility="7.3.000" expanded="true" height="68" name="Transform Cases (2)" width="90" x="246" y="34">
            <parameter key="transform_to" value="lower case"/>
          </operator>
          <operator activated="true" class="text:filter_by_length" compatibility="7.3.000" expanded="true" height="68" name="Filter Tokens (2)" width="90" x="112" y="238">
            <parameter key="min_chars" value="4"/>
            <parameter key="max_chars" value="30"/>
          </operator>
          <operator activated="true" class="text:stem_porter" compatibility="7.3.000" expanded="true" height="68" name="Stem (2)" width="90" x="246" y="238"/>
          <operator activated="true" class="text:stem_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Stem (3)" width="90" x="380" y="136">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\stemmer.txt"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Filter Stopwords (2)" width="90" x="514" y="187">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\gr_stopwords.txt"/>
            <parameter key="case_sensitive" value="false"/>
            <parameter key="encoding" value="SYSTEM"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_english" compatibility="7.3.000" expanded="true" height="68" name="Filter Stopwords (3)" width="90" x="514" y="34"/>
          <operator activated="true" class="text:generate_n_grams_terms" compatibility="7.3.000" expanded="true" height="68" name="Generate n-Grams (2)" width="90" x="648" y="34">
            <parameter key="max_length" value="2"/>
          </operator>
          <connect from_port="document" to_op="Tokenize (2)" to_port="document"/>
          <connect from_op="Tokenize (2)" from_port="document" to_op="Transform Cases (2)" to_port="document"/>
          <connect from_op="Transform Cases (2)" from_port="document" to_op="Filter Tokens (2)" to_port="document"/>
          <connect from_op="Filter Tokens (2)" from_port="document" to_op="Stem (2)" to_port="document"/>
          <connect from_op="Stem (2)" from_port="document" to_op="Stem (3)" to_port="document"/>
          <connect from_op="Stem (3)" from_port="document" to_op="Filter Stopwords (2)" to_port="document"/>
          <connect from_op="Filter Stopwords (2)" from_port="document" to_op="Filter Stopwords (3)" to_port="document"/>
          <connect from_op="Filter Stopwords (3)" from_port="document" to_op="Generate n-Grams (2)" to_port="document"/>
          <connect from_op="Generate n-Grams (2)" from_port="document" to_port="document 1"/>
          <portSpacing port="source_document" spacing="0"/>
          <portSpacing port="sink_document 1" spacing="0"/>
          <portSpacing port="sink_document 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model (2)" width="90" x="581" y="238">
        <list key="application_parameters"/>
        <parameter key="create_view" value="false"/>
      </operator>
      <connect from_op="Read Excel" from_port="output" to_op="Process Documents from Data" to_port="example set"/>
      <connect from_op="Process Documents from Data" from_port="example set" to_op="Set Role" to_port="example set input"/>
      <connect from_op="Process Documents from Data" from_port="word list" to_op="Process Apply Documents" to_port="word list"/>
      <connect from_op="Set Role" from_port="example set output" to_op="Validation" to_port="training"/>
      <connect from_op="Validation" from_port="model" to_op="Apply Model (2)" to_port="model"/>
      <connect from_op="Validation" from_port="training" to_port="result 1"/>
      <connect from_op="Validation" from_port="averagable 1" to_port="result 2"/>
      <connect from_op="Read Excel (2)" from_port="output" to_op="Process Apply Documents" to_port="example set"/>
      <connect from_op="Process Apply Documents" from_port="example set" to_op="Apply Model (2)" to_port="unlabelled data"/>
      <connect from_op="Apply Model (2)" from_port="labelled data" to_port="result 3"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
      <portSpacing port="sink_result 3" spacing="0"/>
      <portSpacing port="sink_result 4" spacing="0"/>
    </process>
  </operator>
</process>

after Process Documents from Data the WordList (Process Documents from Data) result window is empty so it can't continue to the Validation procedure because it hasn't any attribute. I've tried the same process with and without the stem (dictionary) and the problem is with the stemmer of greek words.

Thomas_Ott

The way you have it should work. I wonder if there's a bug in those stemmer/stopword dictionary operators because they ask you enter the file path and name to the txt file.

Try it with an Open File operator attached to them and let's see if that works.

<?xml version="1.0" encoding="UTF-8"?><process version="7.3.001">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="6.0.002" expanded="true" name="Process">
    <process expanded="true">
      <operator activated="true" class="read_excel" compatibility="7.3.001" expanded="true" height="68" name="Read Excel" width="90" x="45" y="34">
        <parameter key="imported_cell_range" value="A1:C1574"/>
        <parameter key="first_row_as_names" value="false"/>
        <list key="annotations">
          <parameter key="0" value="Name"/>
        </list>
        <list key="data_set_meta_data_information">
          <parameter key="0" value="ID.true.numeric.id"/>
          <parameter key="1" value="Tweet.true.text.attribute"/>
          <parameter key="2" value="Sentiment.true.binominal.label"/>
        </list>
      </operator>
      <operator activated="true" class="text:process_document_from_data" compatibility="7.3.000" expanded="true" height="82" name="Process Documents from Data" width="90" x="246" y="34">
        <parameter key="keep_text" value="true"/>
        <parameter key="prune_method" value="percentual"/>
        <parameter key="prune_below_percent" value="1.0"/>
        <parameter key="prune_above_percent" value="62.4"/>
        <parameter key="prune_below_absolute" value="1"/>
        <parameter key="prune_above_absolute" value="10"/>
        <list key="specify_weights"/>
        <process expanded="true">
          <operator activated="true" class="text:tokenize" compatibility="7.3.000" expanded="true" height="68" name="Tokenize" width="90" x="45" y="34"/>
          <operator activated="true" class="text:transform_cases" compatibility="7.3.000" expanded="true" height="68" name="Transform Cases" width="90" x="179" y="34"/>
          <operator activated="true" class="text:filter_by_length" compatibility="7.3.000" expanded="true" height="68" name="Filter Tokens (by Length)" width="90" x="313" y="34">
            <parameter key="max_chars" value="30"/>
          </operator>
          <operator activated="true" class="text:stem_porter" compatibility="7.3.000" expanded="true" height="68" name="Stem (Porter)" width="90" x="447" y="34"/>
          <operator activated="true" class="open_file" compatibility="7.3.001" expanded="true" height="68" name="Open File" width="90" x="246" y="187">
            <parameter key="filename" value="C:\Users\klimi\Desktop\Thesis\stemmer.txt"/>
          </operator>
          <operator activated="true" class="text:stem_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Stem (Dictionary)" width="90" x="514" y="187">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\stemmer.txt"/>
          </operator>
          <operator activated="true" class="open_file" compatibility="7.3.001" expanded="true" height="68" name="Open File (2)" width="90" x="246" y="289">
            <parameter key="filename" value="C:\Users\klimi\Desktop\Thesis\gr_stopwords.txt"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Filter Stopwords (Dictionary)" width="90" x="648" y="289">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\gr_stopwords.txt"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_english" compatibility="7.3.000" expanded="true" height="68" name="Filter Stopwords (English)" width="90" x="782" y="187"/>
          <operator activated="true" class="text:generate_n_grams_terms" compatibility="7.3.000" expanded="true" height="68" name="Generate n-Grams (Terms)" width="90" x="916" y="187"/>
          <connect from_port="document" to_op="Tokenize" to_port="document"/>
          <connect from_op="Tokenize" from_port="document" to_op="Transform Cases" to_port="document"/>
          <connect from_op="Transform Cases" from_port="document" to_op="Filter Tokens (by Length)" to_port="document"/>
          <connect from_op="Filter Tokens (by Length)" from_port="document" to_op="Stem (Porter)" to_port="document"/>
          <connect from_op="Stem (Porter)" from_port="document" to_op="Stem (Dictionary)" to_port="document"/>
          <connect from_op="Open File" from_port="file" to_op="Stem (Dictionary)" to_port="file"/>
          <connect from_op="Stem (Dictionary)" from_port="document" to_op="Filter Stopwords (Dictionary)" to_port="document"/>
          <connect from_op="Open File (2)" from_port="file" to_op="Filter Stopwords (Dictionary)" to_port="file"/>
          <connect from_op="Filter Stopwords (Dictionary)" from_port="document" to_op="Filter Stopwords (English)" to_port="document"/>
          <connect from_op="Filter Stopwords (English)" from_port="document" to_op="Generate n-Grams (Terms)" to_port="document"/>
          <connect from_op="Generate n-Grams (Terms)" from_port="document" to_port="document 1"/>
          <portSpacing port="source_document" spacing="0"/>
          <portSpacing port="sink_document 1" spacing="0"/>
          <portSpacing port="sink_document 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="set_role" compatibility="7.3.001" expanded="true" height="82" name="Set Role" width="90" x="380" y="34">
        <parameter key="attribute_name" value="Sentiment"/>
        <parameter key="target_role" value="label"/>
        <list key="set_additional_roles"/>
      </operator>
      <operator activated="true" class="x_validation" compatibility="7.3.001" expanded="true" height="124" name="Validation" width="90" x="514" y="34">
        <process expanded="true">
          <operator activated="true" class="support_vector_machine" compatibility="7.3.001" expanded="true" height="124" name="SVM (2)" width="90" x="179" y="34">
            <parameter key="kernel_type" value="radial"/>
            <parameter key="kernel_gamma" value="0.0012760559257285525"/>
            <parameter key="C" value="63.095734448019364"/>
          </operator>
          <connect from_port="training" to_op="SVM (2)" to_port="training set"/>
          <connect from_op="SVM (2)" from_port="model" to_port="model"/>
          <portSpacing port="source_training" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true">
          <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model" width="90" x="45" y="34">
            <list key="application_parameters"/>
          </operator>
          <operator activated="true" class="performance" compatibility="7.3.001" expanded="true" height="82" name="Performance (2)" width="90" x="246" y="34"/>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Performance (2)" to_port="labelled data"/>
          <connect from_op="Performance (2)" from_port="performance" to_port="averagable 1"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_averagable 1" spacing="0"/>
          <portSpacing port="sink_averagable 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="read_excel" compatibility="7.3.001" expanded="true" height="68" name="Read Excel (2)" width="90" x="45" y="187">
        <parameter key="imported_cell_range" value="A1:C8990"/>
        <parameter key="first_row_as_names" value="false"/>
        <list key="annotations">
          <parameter key="0" value="Name"/>
        </list>
        <list key="data_set_meta_data_information">
          <parameter key="0" value="ID.true.numeric.id"/>
          <parameter key="1" value="Tweet.true.text.attribute"/>
          <parameter key="2" value="Sentiment.true.attribute_value.attribute"/>
        </list>
      </operator>
      <operator activated="true" class="text:process_document_from_data" compatibility="7.3.000" expanded="true" height="82" name="Process Apply Documents" width="90" x="313" y="187">
        <parameter key="keep_text" value="true"/>
        <parameter key="prune_method" value="percentual"/>
        <parameter key="prune_below_percent" value="1.0"/>
        <parameter key="prune_above_percent" value="62.4"/>
        <parameter key="prune_below_absolute" value="1"/>
        <parameter key="prune_above_absolute" value="10"/>
        <list key="specify_weights"/>
        <process expanded="true">
          <operator activated="true" class="text:tokenize" compatibility="7.3.000" expanded="true" height="68" name="Tokenize (2)" width="90" x="112" y="34"/>
          <operator activated="true" class="text:transform_cases" compatibility="7.3.000" expanded="true" height="68" name="Transform Cases (2)" width="90" x="246" y="34"/>
          <operator activated="true" class="text:filter_by_length" compatibility="7.3.000" expanded="true" height="68" name="Filter Tokens (2)" width="90" x="112" y="238">
            <parameter key="max_chars" value="30"/>
          </operator>
          <operator activated="true" class="text:stem_porter" compatibility="7.3.000" expanded="true" height="68" name="Stem (2)" width="90" x="246" y="238"/>
          <operator activated="true" class="open_file" compatibility="7.3.001" expanded="true" height="68" name="Open File (3)" width="90" x="179" y="340">
            <parameter key="filename" value="C:\Users\klimi\Desktop\Thesis\stemmer.txt"/>
          </operator>
          <operator activated="true" class="text:stem_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Stem (3)" width="90" x="380" y="289">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\stemmer.txt"/>
          </operator>
          <operator activated="true" class="open_file" compatibility="7.3.001" expanded="true" height="68" name="Open File (4)" width="90" x="380" y="391">
            <parameter key="filename" value="C:\Users\klimi\Desktop\Thesis\gr_stopwords.txt"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_dictionary" compatibility="7.3.000" expanded="true" height="82" name="Filter Stopwords (2)" width="90" x="648" y="391">
            <parameter key="file" value="C:\Users\klimi\Desktop\Thesis\gr_stopwords.txt"/>
          </operator>
          <operator activated="true" class="text:filter_stopwords_english" compatibility="7.3.000" expanded="true" height="68" name="Filter Stopwords (3)" width="90" x="514" y="34"/>
          <operator activated="true" class="text:generate_n_grams_terms" compatibility="7.3.000" expanded="true" height="68" name="Generate n-Grams (2)" width="90" x="648" y="34"/>
          <connect from_port="document" to_op="Tokenize (2)" to_port="document"/>
          <connect from_op="Tokenize (2)" from_port="document" to_op="Transform Cases (2)" to_port="document"/>
          <connect from_op="Transform Cases (2)" from_port="document" to_op="Filter Tokens (2)" to_port="document"/>
          <connect from_op="Filter Tokens (2)" from_port="document" to_op="Stem (2)" to_port="document"/>
          <connect from_op="Stem (2)" from_port="document" to_op="Stem (3)" to_port="document"/>
          <connect from_op="Open File (3)" from_port="file" to_op="Stem (3)" to_port="file"/>
          <connect from_op="Stem (3)" from_port="document" to_op="Filter Stopwords (2)" to_port="document"/>
          <connect from_op="Open File (4)" from_port="file" to_op="Filter Stopwords (2)" to_port="file"/>
          <connect from_op="Filter Stopwords (2)" from_port="document" to_op="Filter Stopwords (3)" to_port="document"/>
          <connect from_op="Filter Stopwords (3)" from_port="document" to_op="Generate n-Grams (2)" to_port="document"/>
          <connect from_op="Generate n-Grams (2)" from_port="document" to_port="document 1"/>
          <portSpacing port="source_document" spacing="0"/>
          <portSpacing port="sink_document 1" spacing="0"/>
          <portSpacing port="sink_document 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="apply_model" compatibility="7.1.001" expanded="true" height="82" name="Apply Model (2)" width="90" x="581" y="238">
        <list key="application_parameters"/>
      </operator>
      <connect from_op="Read Excel" from_port="output" to_op="Process Documents from Data" to_port="example set"/>
      <connect from_op="Process Documents from Data" from_port="example set" to_op="Set Role" to_port="example set input"/>
      <connect from_op="Process Documents from Data" from_port="word list" to_op="Process Apply Documents" to_port="word list"/>
      <connect from_op="Set Role" from_port="example set output" to_op="Validation" to_port="training"/>
      <connect from_op="Validation" from_port="model" to_op="Apply Model (2)" to_port="model"/>
      <connect from_op="Validation" from_port="training" to_port="result 1"/>
      <connect from_op="Validation" from_port="averagable 1" to_port="result 2"/>
      <connect from_op="Read Excel (2)" from_port="output" to_op="Process Apply Documents" to_port="example set"/>
      <connect from_op="Process Apply Documents" from_port="example set" to_op="Apply Model (2)" to_port="unlabelled data"/>
      <connect from_op="Apply Model (2)" from_port="labelled data" to_port="result 3"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
      <portSpacing port="sink_result 3" spacing="0"/>
      <portSpacing port="sink_result 4" spacing="0"/>
    </process>
  </operator>
</process>

slimik

with your solution the process bypasses the previous error that i mention. But still the stemmer doesn't work no matter what rule i give. Is there any way to implemet a python based stemmer as an rapidminer operator?

IngoRM

Should be possible but might require some work on your side of course. You need to install the Python extension from the Marketplace (https://marketplace.rapidminer.com/UpdateServer/faces/product_details.xhtml?productId=rmx_python_scripting) and then implement the function yourself.

Cheers,

Ingo

slimik

ok. Thank you i'll give it a try!

todimary

Hello, any news with your work?