"Search for a phrase in multiples PDFs via a list of URLs"

carl
carl New Altair Community Member
edited November 5 in Community Q&A

Is it possible to input a list of URLs (which contain PDFs), then search for a phrase in all the PDFs, and return a table with the URL path and the searched-for text?  I'd like to do this without downloading all the PDFs.

 

This is what I have so far, which runs, but doesn't create an attribute.

 

1 - Read Excel (with URLs)

2 - Loop Examples

2a - Extract Macro

2b - Open File

2c - Read Document

2d - Extract Information

3 - Select Attributes

 

<?xml version="1.0" encoding="UTF-8"?><process version="7.3.000">
  <context>
    <input/>
    <output/>
    <macros/>
  </context>
  <operator activated="true" class="process" compatibility="7.3.000" expanded="true" name="Process">
    <process expanded="true">
      <operator activated="true" class="read_excel" compatibility="7.3.000" expanded="true" height="68" name="Read Excel" width="90" x="179" y="34">
        <parameter key="excel_file" value="/Users/carl/Documents/SD PDFs.xlsx"/>
        <parameter key="imported_cell_range" value="A"/>
        <list key="annotations">
          <parameter key="0" value="Name"/>
        </list>
        <parameter key="locale" value="English (United Kingdom)"/>
        <list key="data_set_meta_data_information">
          <parameter key="0" value="SvceDef Link.true.file_path.attribute"/>
        </list>
      </operator>
      <operator activated="true" class="loop_examples" compatibility="7.3.000" expanded="true" height="103" name="Loop Examples" width="90" x="313" y="34">
        <process expanded="true">
          <operator activated="true" class="extract_macro" compatibility="7.3.000" expanded="true" height="68" name="Extract Macro" width="90" x="45" y="34">
            <parameter key="macro" value="GetURL"/>
            <parameter key="macro_type" value="data_value"/>
            <parameter key="attribute_name" value="SvceDef Link"/>
            <parameter key="example_index" value="%{example}"/>
            <list key="additional_macros"/>
          </operator>
          <operator activated="true" class="open_file" compatibility="7.3.000" expanded="true" height="68" name="Open File" width="90" x="179" y="34">
            <parameter key="resource_type" value="URL"/>
            <parameter key="url" value="%{GetURL}"/>
          </operator>
          <operator activated="true" class="text:read_document" compatibility="7.3.000" expanded="true" height="68" name="Read Document" width="90" x="313" y="34">
            <parameter key="content_type" value="pdf"/>
          </operator>
          <operator activated="true" class="text:extract_information" compatibility="7.3.000" expanded="true" height="68" name="Extract Information" width="90" x="447" y="34">
            <parameter key="query_type" value="Regular Expression"/>
            <list key="string_machting_queries"/>
            <list key="regular_expression_queries">
              <parameter key="Text" value="sample text"/>
            </list>
            <list key="regular_region_queries"/>
            <list key="xpath_queries"/>
            <list key="namespaces"/>
            <list key="index_queries"/>
            <list key="jsonpath_queries"/>
          </operator>
          <connect from_port="example set" to_op="Extract Macro" to_port="example set"/>
          <connect from_op="Open File" from_port="file" to_op="Read Document" to_port="file"/>
          <connect from_op="Read Document" from_port="output" to_op="Extract Information" to_port="document"/>
          <connect from_op="Extract Information" from_port="document" to_port="output 1"/>
          <portSpacing port="source_example set" spacing="0"/>
          <portSpacing port="sink_example set" spacing="0"/>
          <portSpacing port="sink_output 1" spacing="0"/>
          <portSpacing port="sink_output 2" spacing="0"/>
        </process>
      </operator>
      <operator activated="true" class="select_attributes" compatibility="7.3.000" expanded="true" height="82" name="Select Attributes" width="90" x="447" y="34">
        <parameter key="attribute_filter_type" value="single"/>
        <parameter key="attribute" value="Text"/>
      </operator>
      <connect from_op="Read Excel" from_port="output" to_op="Loop Examples" to_port="example set"/>
      <connect from_op="Loop Examples" from_port="example set" to_op="Select Attributes" to_port="example set input"/>
      <connect from_op="Select Attributes" from_port="example set output" to_port="result 1"/>
      <portSpacing port="source_input 1" spacing="0"/>
      <portSpacing port="sink_result 1" spacing="0"/>
      <portSpacing port="sink_result 2" spacing="0"/>
    </process>
  </operator>
</process>
Tagged:

Best Answer

  • JEdward
    JEdward New Altair Community Member
    Answer ✓

    Here you go.  You needed a Documents to Data operator to change your PDF text into an ExampleSet. 

     

    <?xml version="1.0" encoding="UTF-8"?><process version="7.3.000">
      <context>
        <input/>
        <output/>
        <macros/>
      </context>
      <operator activated="true" class="process" compatibility="7.3.000" expanded="true" name="Process">
        <process expanded="true">
          <operator activated="true" class="read_excel" compatibility="7.3.000" expanded="true" height="68" name="Read Excel" width="90" x="179" y="34">
            <parameter key="excel_file" value="/Users/carl/Documents/SD PDFs.xlsx"/>
            <parameter key="imported_cell_range" value="A"/>
            <list key="annotations">
              <parameter key="0" value="Name"/>
            </list>
            <parameter key="locale" value="English (United Kingdom)"/>
            <list key="data_set_meta_data_information">
              <parameter key="0" value="SvceDef Link.true.file_path.attribute"/>
            </list>
          </operator>
          <operator activated="true" class="loop_examples" compatibility="7.3.000" expanded="true" height="103" name="Loop Examples" width="90" x="313" y="34">
            <process expanded="true">
              <operator activated="true" class="extract_macro" compatibility="7.3.000" expanded="true" height="68" name="Extract Macro" width="90" x="45" y="34">
                <parameter key="macro" value="GetURL"/>
                <parameter key="macro_type" value="data_value"/>
                <parameter key="attribute_name" value="SvceDef Link"/>
                <parameter key="example_index" value="%{example}"/>
                <list key="additional_macros"/>
              </operator>
              <operator activated="true" class="open_file" compatibility="7.3.000" expanded="true" height="68" name="Open File" width="90" x="179" y="34">
                <parameter key="resource_type" value="URL"/>
                <parameter key="url" value="%{GetURL}"/>
              </operator>
              <operator activated="true" class="text:read_document" compatibility="7.3.000" expanded="true" height="68" name="Read Document" width="90" x="313" y="34">
                <parameter key="content_type" value="pdf"/>
              </operator>
              <operator activated="true" class="text:extract_information" compatibility="7.3.000" expanded="true" height="68" name="Extract Information" width="90" x="447" y="34">
                <parameter key="query_type" value="Regular Expression"/>
                <list key="string_machting_queries"/>
                <list key="regular_expression_queries">
                  <parameter key="Text" value="sample text"/>
                </list>
                <list key="regular_region_queries"/>
                <list key="xpath_queries"/>
                <list key="namespaces"/>
                <list key="index_queries"/>
                <list key="jsonpath_queries"/>
              </operator>
              <operator activated="true" class="text:documents_to_data" compatibility="7.3.000" expanded="true" height="82" name="Documents to Data" width="90" x="648" y="34">
                <parameter key="text_attribute" value="OriginalText"/>
              </operator>
              <connect from_port="example set" to_op="Extract Macro" to_port="example set"/>
              <connect from_op="Open File" from_port="file" to_op="Read Document" to_port="file"/>
              <connect from_op="Read Document" from_port="output" to_op="Extract Information" to_port="document"/>
              <connect from_op="Extract Information" from_port="document" to_op="Documents to Data" to_port="documents 1"/>
              <connect from_op="Documents to Data" from_port="example set" to_port="output 1"/>
              <portSpacing port="source_example set" spacing="0"/>
              <portSpacing port="sink_example set" spacing="0"/>
              <portSpacing port="sink_output 1" spacing="0"/>
              <portSpacing port="sink_output 2" spacing="0"/>
            </process>
          </operator>
          <operator activated="true" class="append" compatibility="7.3.000" expanded="true" height="82" name="Append" width="90" x="514" y="85"/>
          <operator activated="true" class="select_attributes" compatibility="7.3.000" expanded="true" height="82" name="Select Attributes" width="90" x="715" y="34">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Text"/>
          </operator>
          <connect from_op="Read Excel" from_port="output" to_op="Loop Examples" to_port="example set"/>
          <connect from_op="Loop Examples" from_port="output 1" to_op="Append" to_port="example set 1"/>
          <connect from_op="Append" from_port="merged set" to_op="Select Attributes" to_port="example set input"/>
          <connect from_op="Select Attributes" from_port="example set output" to_port="result 1"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="sink_result 1" spacing="0"/>
          <portSpacing port="sink_result 2" spacing="0"/>
        </process>
      </operator>
    </process>

Answers

  • JEdward
    JEdward New Altair Community Member
    Answer ✓

    Here you go.  You needed a Documents to Data operator to change your PDF text into an ExampleSet. 

     

    <?xml version="1.0" encoding="UTF-8"?><process version="7.3.000">
      <context>
        <input/>
        <output/>
        <macros/>
      </context>
      <operator activated="true" class="process" compatibility="7.3.000" expanded="true" name="Process">
        <process expanded="true">
          <operator activated="true" class="read_excel" compatibility="7.3.000" expanded="true" height="68" name="Read Excel" width="90" x="179" y="34">
            <parameter key="excel_file" value="/Users/carl/Documents/SD PDFs.xlsx"/>
            <parameter key="imported_cell_range" value="A"/>
            <list key="annotations">
              <parameter key="0" value="Name"/>
            </list>
            <parameter key="locale" value="English (United Kingdom)"/>
            <list key="data_set_meta_data_information">
              <parameter key="0" value="SvceDef Link.true.file_path.attribute"/>
            </list>
          </operator>
          <operator activated="true" class="loop_examples" compatibility="7.3.000" expanded="true" height="103" name="Loop Examples" width="90" x="313" y="34">
            <process expanded="true">
              <operator activated="true" class="extract_macro" compatibility="7.3.000" expanded="true" height="68" name="Extract Macro" width="90" x="45" y="34">
                <parameter key="macro" value="GetURL"/>
                <parameter key="macro_type" value="data_value"/>
                <parameter key="attribute_name" value="SvceDef Link"/>
                <parameter key="example_index" value="%{example}"/>
                <list key="additional_macros"/>
              </operator>
              <operator activated="true" class="open_file" compatibility="7.3.000" expanded="true" height="68" name="Open File" width="90" x="179" y="34">
                <parameter key="resource_type" value="URL"/>
                <parameter key="url" value="%{GetURL}"/>
              </operator>
              <operator activated="true" class="text:read_document" compatibility="7.3.000" expanded="true" height="68" name="Read Document" width="90" x="313" y="34">
                <parameter key="content_type" value="pdf"/>
              </operator>
              <operator activated="true" class="text:extract_information" compatibility="7.3.000" expanded="true" height="68" name="Extract Information" width="90" x="447" y="34">
                <parameter key="query_type" value="Regular Expression"/>
                <list key="string_machting_queries"/>
                <list key="regular_expression_queries">
                  <parameter key="Text" value="sample text"/>
                </list>
                <list key="regular_region_queries"/>
                <list key="xpath_queries"/>
                <list key="namespaces"/>
                <list key="index_queries"/>
                <list key="jsonpath_queries"/>
              </operator>
              <operator activated="true" class="text:documents_to_data" compatibility="7.3.000" expanded="true" height="82" name="Documents to Data" width="90" x="648" y="34">
                <parameter key="text_attribute" value="OriginalText"/>
              </operator>
              <connect from_port="example set" to_op="Extract Macro" to_port="example set"/>
              <connect from_op="Open File" from_port="file" to_op="Read Document" to_port="file"/>
              <connect from_op="Read Document" from_port="output" to_op="Extract Information" to_port="document"/>
              <connect from_op="Extract Information" from_port="document" to_op="Documents to Data" to_port="documents 1"/>
              <connect from_op="Documents to Data" from_port="example set" to_port="output 1"/>
              <portSpacing port="source_example set" spacing="0"/>
              <portSpacing port="sink_example set" spacing="0"/>
              <portSpacing port="sink_output 1" spacing="0"/>
              <portSpacing port="sink_output 2" spacing="0"/>
            </process>
          </operator>
          <operator activated="true" class="append" compatibility="7.3.000" expanded="true" height="82" name="Append" width="90" x="514" y="85"/>
          <operator activated="true" class="select_attributes" compatibility="7.3.000" expanded="true" height="82" name="Select Attributes" width="90" x="715" y="34">
            <parameter key="attribute_filter_type" value="single"/>
            <parameter key="attribute" value="Text"/>
          </operator>
          <connect from_op="Read Excel" from_port="output" to_op="Loop Examples" to_port="example set"/>
          <connect from_op="Loop Examples" from_port="output 1" to_op="Append" to_port="example set 1"/>
          <connect from_op="Append" from_port="merged set" to_op="Select Attributes" to_port="example set input"/>
          <connect from_op="Select Attributes" from_port="example set output" to_port="result 1"/>
          <portSpacing port="source_input 1" spacing="0"/>
          <portSpacing port="sink_result 1" spacing="0"/>
          <portSpacing port="sink_result 2" spacing="0"/>
        </process>
      </operator>
    </process>
  • carl
    carl New Altair Community Member

    Thanks so much!  Works perfectly.

Categories