Loop files bug?
Hi all,
I'm trying to use a loop files operator and append over a series of .csv files. However, when trying to append I get an error message that the attribute 'Telephone1' does not have the same data type in all my files, which is correct (in some it is nominal while in other a real value). I therefore added the 'parse numbers' operator into the loop files to change the data type of 'Telephone1', but when I do that, I get an error that the attribute does not exist.
-I've checked with breakpoints, but the attribute does exist in every uploaded .csv file
-If tried if there was somehow a whitespace after the variable which may have caused the problem, this is not the case (even if I copy the attribute name from my results view, I get the same error).
-I have tried to use other operators to see if they worked: if I do a 'select attributes' with 'Telephone1' RM does not give an error message and the attribute is select and gives a valid output, however, if I used the branch operator (and in it the parse numbers) because I only want the numbers parsed if 'Telephone1' is recognised as a nominal value it doesn't work either.
Anyone has an idea what might be the issue? I've included my XML below.
<operator activated="true" class="parse_numbers" compatibility="7.5.003" expanded="true" height="82" name="Parse Numbers" width="90" x="447" y="34">
<parameter key="attribute_filter_type" value="subset"/>
<parameter key="attributes" value="Telephone2|Mobile2|Telephone1|Mobile1"/>
</operator>
<connect from_port="file object" to_op="Read CSV" to_port="file"/>
<connect from_op="Read CSV" from_port="output" to_op="Generate Attributes" to_port="example set input"/>
<connect from_op="Generate Attributes" from_port="example set output" to_op="Parse Numbers" to_port="example set input"/>
<connect from_op="Parse Numbers" from_port="example set output" to_port="output 1"/>
<portSpacing port="source_file object" spacing="0"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
<description align="center" color="transparent" colored="false" width="126">Getting the data from SFTP of agency</description>
</operator>
<operator activated="true" breakpoints="after" class="append" compatibility="7.5.003" expanded="true" height="82" name="Append" width="90" x="246" y="289"/>
<operator activated="true" breakpoints="after" class="select_subprocess" compatibility="7.5.003" expanded="true" height="82" name="Check uploaded" width="90" x="380" y="289">
<process expanded="true">
<operator activated="true" class="jdbc_connectors:read_database" compatibility="7.5.003" expanded="true" height="68" name="Read Database" width="90" x="45" y="136">
<parameter key="connection" value="DDmachine2"/>
<parameter key="define_query" value="table name"/>
<parameter key="table_name" value="DDstreetimport_results_handled"/>
<enumeration key="parameters"/>
</operator>
<operator activated="true" class="filter_examples" compatibility="7.5.003" expanded="true" height="103" name="Filter Examples" width="90" x="179" y="136">
<list key="filters_list">
<parameter key="filters_entry_key" value="DDstreetimport_datasource_typename.equals."DR csv Streetimport""/>
</list>
</operator>
<operator activated="true" class="select_attributes" compatibility="7.5.003" expanded="true" height="82" name="Select Attributes" width="90" x="313" y="136">
<parameter key="attribute_filter_type" value="subset"/>
<parameter key="attributes" value="DDstreetimport_datasource_typename|DDstreetimport_datasource_batchname"/>
</operator>
<operator activated="true" class="generate_attributes" compatibility="7.5.003" expanded="true" height="82" name="Generate Attributes (2)" width="90" x="447" y="136">
<list key="function_descriptions">
<parameter key="already_upload" value=""Y""/>
</list>
</operator>
<operator activated="true" class="join" compatibility="7.5.003" expanded="true" height="82" name="Join" width="90" x="581" y="34">
<parameter key="join_type" value="left"/>
<parameter key="use_id_attribute_as_key" value="false"/>
<list key="key_attributes">
<parameter key="ddstreetimport_datasource_batchname" value="DDstreetimport_datasource_batchname"/>
<parameter key="ddstreetimport_datasource_typename" value="DDstreetimport_datasource_typename"/>
</list>
</operator>
<operator activated="true" class="filter_examples" compatibility="7.5.003" expanded="true" height="103" name="Filter Examples (2)" width="90" x="715" y="34">
<list key="filters_list">
<parameter key="filters_entry_key" value="already_upload.does_not_equal.Y"/>
</list>
</operator>
<operator activated="true" class="select_attributes" compatibility="7.5.003" expanded="true" height="82" name="Select Attributes (2)" width="90" x="849" y="34">
<parameter key="attribute_filter_type" value="single"/>
<parameter key="attribute" value="already_upload"/>
<parameter key="invert_selection" value="true"/>
</operator>
<connect from_port="input 1" to_op="Join" to_port="left"/>
<connect from_op="Read Database" from_port="output" to_op="Filter Examples" to_port="example set input"/>
<connect from_op="Filter Examples" from_port="example set output" to_op="Select Attributes" to_port="example set input"/>
<connect from_op="Select Attributes" from_port="example set output" to_op="Generate Attributes (2)" to_port="example set input"/>
<connect from_op="Generate Attributes (2)" from_port="original" to_op="Join" to_port="right"/>
<connect from_op="Join" from_port="join" to_op="Filter Examples (2)" to_port="example set input"/>
<connect from_op="Filter Examples (2)" from_port="example set output" to_op="Select Attributes (2)" to_port="example set input"/>
<connect from_op="Select Attributes (2)" from_port="example set output" to_port="output 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="source_input 2" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
<process expanded="true">
<connect from_port="input 1" to_port="output 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="source_input 2" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
<description align="center" color="transparent" colored="false" width="126">1: check if already upload and take only not upload files.<br>2: do not check and take all files.</description>
</operator>
<operator activated="true" class="select_subprocess" compatibility="7.5.003" expanded="true" height="82" name="Cleaning data" width="90" x="715" y="289">
<process expanded="true">
<operator activated="true" class="subprocess" compatibility="7.5.003" expanded="true" height="82" name="data cleaning" width="90" x="45" y="34">
<process expanded="true">
<portSpacing port="source_in 1" spacing="0"/>
<portSpacing port="source_in 2" spacing="0"/>
<portSpacing port="sink_out 1" spacing="0"/>
<portSpacing port="sink_out 2" spacing="0"/>
</process>
<description align="center" color="transparent" colored="false" width="126">Cleansing some data issues as they come from the streetimport</description>
</operator>
<operator activated="true" class="subprocess" compatibility="7.5.003" expanded="true" height="82" name="internal resultcodes" width="90" x="179" y="34">
<process expanded="true">
<portSpacing port="source_in 1" spacing="0"/>
<portSpacing port="source_in 2" spacing="0"/>
<portSpacing port="sink_out 1" spacing="0"/>
<portSpacing port="sink_out 2" spacing="0"/>
</process>
</operator>
<operator activated="true" class="subprocess" compatibility="7.5.003" expanded="true" height="82" name="update contact_id" width="90" x="313" y="34">
<process expanded="true">
<portSpacing port="source_in 1" spacing="0"/>
<portSpacing port="source_in 2" spacing="0"/>
<portSpacing port="sink_out 1" spacing="0"/>
<portSpacing port="sink_out 2" spacing="0"/>
</process>
</operator>
<connect from_port="input 1" to_op="data cleaning" to_port="in 1"/>
<connect from_op="data cleaning" from_port="out 1" to_op="internal resultcodes" to_port="in 1"/>
<connect from_op="internal resultcodes" from_port="out 1" to_op="update contact_id" to_port="in 1"/>
<connect from_op="update contact_id" from_port="out 1" to_port="output 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="source_input 2" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
<process expanded="true">
<connect from_port="input 1" to_port="output 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="source_input 2" spacing="0"/>
<portSpacing port="sink_output 1" spacing="0"/>
<portSpacing port="sink_output 2" spacing="0"/>
</process>
<description align="center" color="transparent" colored="false" width="126">cleans data from streetimport, only if it is new data</description>
</operator>
<connect from_port="input 1" to_op="Set Macros" to_port="through 1"/>
<connect from_op="Set Macros" from_port="through 1" to_op="Set Macro (2)" to_port="through 1"/>
<connect from_op="Loop Files" from_port="output 1" to_op="Append" to_port="example set 1"/>
<connect from_op="Append" from_port="merged set" to_op="Check uploaded" to_port="input 1"/>
<connect from_op="Check uploaded" from_port="output 1" to_op="Cleaning data" to_port="input 1"/>
<portSpacing port="source_input 1" spacing="0"/>
<portSpacing port="source_input 2" spacing="0"/>
<portSpacing port="sink_result 1" spacing="0"/>
<description align="center" color="blue" colored="true" height="50" resized="true" width="403" x="428" y="30">Getting and processing streetimport files we get from agency</description>
<description align="center" color="blue" colored="true" height="249" resized="true" width="166" x="10" y="254">Upload from agency</description>
<description align="center" color="green" colored="true" height="248" resized="true" width="654" x="179" y="254">Preprocessing</description>
<description align="center" color="orange" colored="true" height="252" resized="true" width="364" x="836" y="251">Loading to RMserver</description>
</process>
</operator>
</process>