How to use fields created from xpath extraction?

luiz_vidal

Guys,

Again I need your help here. It's seems simple to build it but I didn't figure out yet how.

 

First of all, to give a context to it, I crawled some web pages and extract exact piece of information I wanted via xpath.

As you can see below Preço and Descrição were extracted from the webpage I crawled.

Now I want to use both fields for my data mining process, such as k-NN, random forest, etc.

But right after my operator that process documents from files I want to set role, nominal to text, etc... but I don't see these fields mentioned above.

 

<?xml version="1.0" encoding="UTF-8"?><process version="8.0.001">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="8.0.001" expanded="true" name="Process">
    <process expanded="true">
      <operator activated="false" class="concurrency:loop" compatibility="8.0.001" expanded="true" height="103" name="Loop" width="90" x="112" y="34">
        <parameter key="iteration_macro" value="page"/>
        <process expanded="true">
          <operator activated="true" class="web:crawl_web_modern" compatibility="7.3.000" expanded="true" height="68" name="Crawl Web" width="90" x="246" y="238">
            <parameter key="url" value="https://www.buscape.com.br/cerveja?pagina=%{page}"/>
            <list key="crawling_rules">
              <parameter key="follow_link_with_matching_url" value="cerveja"/>
            </list>
            <parameter key="retrieve_as_html" value="true"/>
            <parameter key="add_content_as_attribute" value="true"/>
            <parameter key="write_pages_to_disk" value="true"/>
            <parameter key="output_dir" value="C:\Users\luiz.vidal\Desktop\Cloudera\SEFA-PA\Cerveja\"/>
          </operator>
          <operator activated="true" class="rename_file" compatibility="8.0.001" expanded="true" height="82" name="Rename File" width="90" x="514" y="238">
            <parameter key="file" value="C:\Users\luiz.vidal\Desktop\Cloudera\SEFA-PA\Cerveja\0.txt"/>
            <parameter key="new_name" value="%{page}.txt"/>
          </operator>
          <operator activated="true" class="generate_macro" compatibility="8.0.001" expanded="true" height="82" name="Generate Macro" width="90" x="112" y="34">
            <list key="function_descriptions">
              <parameter key="page" value="%{page} + 1"/>
            </list>
          </operator>
          <connect from_port="input 1" to_op="Generate Macro" to_port="through 1"/>
          <connect from_op="Crawl Web" from_port="example set" to_op="Rename File" to_port="through 1"/>
          <connect from_op="Rename File" from_port="through 1" to_port="output 2"/>
          <connect from_op="Generate Macro" from_port="through 1" to_port="output 1"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="source_input 2" spacing="0"/>
          <portSpacing port="sink_output 1" spacing="0"/>
          <portSpacing port="sink_output 2" spacing="0"/>
          <portSpacing port="sink_output 3" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="text:process_document_from_file" compatibility="8.1.000" expanded="true" height="82" name="Process Documents from Files" width="90" x="112" y="238">
        <list key="text_directories">
          <parameter key="cerveja" value="C:\Users\luiz.vidal\Desktop\Cloudera\SEFA-PA\Cerveja"/>
        </list>
        <parameter key="encoding" value="ISO-8859-1"/>
        <parameter key="create_word_vector" value="false"/>
        <process expanded="true">
          <operator activated="true" class="text:cut_document" compatibility="8.1.000" expanded="true" height="68" name="Cut Document" width="90" x="447" y="34">
            <parameter key="query_type" value="XPath"/>
            <list key="string_machting_queries"/>
            <list key="regular_expression_queries"/>
            <list key="regular_region_queries"/>
            <list key="xpath_queries">
              <parameter key="Div" value="(//*[@data-trackcheckoutcontainer=&amp;quot;true&quot;])"/>
            </list>
            <list key="namespaces"/>
            <list key="index_queries"/>
            <list key="jsonpath_queries"/>
            <process expanded="true">
              <operator activated="true" class="text:extract_information" compatibility="8.1.000" expanded="true" height="68" name="Extract Price and Name" width="90" x="380" y="85">
                <parameter key="query_type" value="XPath"/>
                <list key="string_machting_queries"/>
                <list key="regular_expression_queries"/>
                <list key="regular_region_queries"/>
                <list key="xpath_queries">
                  <parameter key="Preço" value="//*[@name=&amp;quot;priceProduct&quot;]/@value"/&gt;
                  <parameter key="Descrição" value="//*[@name=&amp;quot;productName&quot;]/@value"/&gt;
                </list>
                <list key="namespaces"/>
                <list key="index_queries"/>
                <list key="jsonpath_queries"/>
              </operator>
              <connect from_port="segment" to_op="Extract Price and Name" to_port="document"/>
              <connect from_op="Extract Price and Name" from_port="document" to_port="document 1"/>
              <portSpacing port="source_segment" spacing="0"/>
              <portSpacing port="sink_document 1" spacing="0"/>
              <portSpacing port="sink_document 2" spacing="0"/>
            </process>
          </operator>
          <connect from_port="document" to_op="Cut Document" to_port="document"/>
          <connect from_op="Cut Document" from_port="documents" to_port="document 1"/>
          <portSpacing port="source_document" spacing="0"/>
          <portSpacing port="sink_document 1" spacing="0"/>
          <portSpacing port="sink_document 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="false" class="select_attributes" compatibility="8.0.001" expanded="true" height="82" name="Select Attributes" width="90" x="313" y="340">
        <parameter key="include_special_attributes" value="true"/>
      </operator>
      <operator activated="false" class="nominal_to_text" compatibility="8.0.001" expanded="true" height="82" name="Nominal to Text" width="90" x="447" y="340">
        <parameter key="include_special_attributes" value="true"/>
      </operator>
      <operator activated="false" class="set_role" compatibility="8.0.001" expanded="true" height="82" name="Set Role" width="90" x="581" y="340">
        <parameter key="attribute_name" value="metadata_path"/>
        <parameter key="target_role" value="label"/>
        <list key="set_additional_roles"/>
      </operator>
      <operator activated="false" class="concurrency:cross_validation" compatibility="8.0.001" expanded="true" height="145" name="Cross Validation" width="90" x="715" y="340">
        <process expanded="true">
          <operator activated="true" class="k_nn" compatibility="8.0.001" expanded="true" height="82" name="k-NN" width="90" x="179" y="34"/>
          <connect from_port="training set" to_op="k-NN" to_port="training set"/>
          <connect from_op="k-NN" from_port="model" to_port="model"/>
          <portSpacing port="source_training set" spacing="0"/>
          <portSpacing port="sink_model" spacing="0"/>
          <portSpacing port="sink_through 1" spacing="0"/>
        </process>
        <process expanded="true">
          <operator activated="true" class="apply_model" compatibility="8.0.001" expanded="true" height="82" name="Apply Model" width="90" x="112" y="34">
            <list key="application_parameters"/>
          </operator>
          <operator activated="true" class="performance" compatibility="8.0.001" expanded="true" height="82" name="Performance" width="90" x="246" y="34"/>
          <connect from_port="model" to_op="Apply Model" to_port="model"/>
          <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
          <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
          <connect from_op="Performance" from_port="performance" to_port="performance 1"/>
          <connect from_op="Performance" from_port="example set" to_port="test set results"/>
          <portSpacing port="source_model" spacing="0"/>
          <portSpacing port="source_test set" spacing="0"/>
          <portSpacing port="source_through 1" spacing="0"/>
          <portSpacing port="sink_test set results" spacing="0"/>
          <portSpacing port="sink_performance 1" spacing="0"/>
          <portSpacing port="sink_performance 2" spacing="0"/>
        </process>
      </operator>
      <connect from_op="Process Documents from Files" from_port="example set" to_port="result 1"/>
      <connect from_op="Select Attributes" from_port="example set output" to_op="Nominal to Text" to_port="example set input"/>
      <connect from_op="Nominal to Text" from_port="example set output" to_op="Set Role" to_port="example set input"/>
      <connect from_op="Set Role" from_port="example set output" to_op="Cross Validation" to_port="example set"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="source_input 2" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>

Would someone give me a little help?

Thanks in advance.

Thomas_Ott

Just toggle on the "Keep Text" in the second Process Documents operator.