Optimal SVM parameters but very different results?

User: "Sasch"
New Altair Community Member
Updated by Jocelyn
Hi all,
I am using the grid search parameter optimizer to determine the best parameters (C and gamma) for my SVM. The SVM is embedded in a 10-fold-validation.
After the process is finished I get the parameter set and a performance of 100 (!)%. (see code 1)
Code 1:

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.2.008">
 <context>
   <input/>
   <output/>
   <macros/>
 </context>
 <operator activated="true" class="process" compatibility="5.2.008" expanded="true" name="Process">
   <process expanded="true" height="467" width="748">
     <operator activated="true" class="read_excel" compatibility="5.2.008" expanded="true" height="60" name="Read Excel" width="90" x="45" y="30">
       <parameter key="excel_file" value="D:\PSY-DATA\06_HERZRATEN_PROJEKT\HR_KlassDaten.xlsx"/>
       <parameter key="imported_cell_range" value="A1:M4901"/>
       <parameter key="first_row_as_names" value="false"/>
       <list key="annotations">
         <parameter key="0" value="Name"/>
       </list>
       <list key="data_set_meta_data_information">
         <parameter key="0" value="Probandennummer.false.integer.attribute"/>
         <parameter key="1" value="Alter.false.integer.attribute"/>
         <parameter key="2" value="Altersgruppe.false.integer.attribute"/>
         <parameter key="3" value="Geschlecht.false.integer.attribute"/>
         <parameter key="4" value="Geschlechtsfaktor.false.integer.attribute"/>
         <parameter key="5" value="Mens.false.integer.attribute"/>
         <parameter key="6" value="RMSSD(ms).true.real.attribute"/>
         <parameter key="7" value="mean_RR(ms).true.real.attribute"/>
         <parameter key="8" value="std_RR(ms).true.real.attribute"/>
         <parameter key="9" value="mean_HR.true.real.attribute"/>
         <parameter key="10" value="std_HR.true.real.attribute"/>
         <parameter key="11" value="label.false.polynominal.attribute"/>
         <parameter key="12" value="label valenz.true.polynominal.label"/>
       </list>
     </operator>
     <operator activated="true" class="optimize_parameters_grid" compatibility="5.2.008" expanded="true" height="94" name="Optimize Parameters (Grid)" width="90" x="447" y="30">
       <list key="parameters">
         <parameter key="SVM.C" value="0.03125,0.125,0.5,2,8,32,128,512,2048,8192,32768"/>
         <parameter key="SVM.gamma" value="0.000030517578125,0.00012207,0.000488281,0.001953125,0.0078125,0.03125,0.125,0.5,2,8"/>
       </list>
       <process expanded="true" height="487" width="826">
         <operator activated="true" class="x_validation" compatibility="5.1.002" expanded="true" height="112" name="Validation" width="90" x="179" y="75">
           <parameter key="average_performances_only" value="false"/>
           <process expanded="true" height="487" width="346">
             <operator activated="true" class="support_vector_machine_libsvm" compatibility="5.2.008" expanded="true" height="76" name="SVM" width="90" x="246" y="30">
               <parameter key="gamma" value="0.001953125"/>
               <parameter key="C" value="32768"/>
               <parameter key="cache_size" value="250"/>
               <list key="class_weights"/>
               <parameter key="calculate_confidences" value="true"/>
             </operator>
             <connect from_port="training" to_op="SVM" to_port="training set"/>
             <connect from_op="SVM" from_port="model" to_port="model"/>
             <portSpacing port="source_training" spacing="0"/>
             <portSpacing port="sink_model" spacing="0"/>
             <portSpacing port="sink_through 1" spacing="0"/>
           </process>
           <process expanded="true" height="487" width="300">
             <operator activated="true" class="apply_model" compatibility="5.2.008" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
               <list key="application_parameters"/>
             </operator>
             <operator activated="true" class="performance_classification" compatibility="5.2.008" expanded="true" height="76" name="Performance" width="90" x="180" y="30">
               <list key="class_weights"/>
             </operator>
             <connect from_port="model" to_op="Apply Model" to_port="model"/>
             <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
             <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
             <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
             <portSpacing port="source_model" spacing="0"/>
             <portSpacing port="source_test set" spacing="0"/>
             <portSpacing port="source_through 1" spacing="0"/>
             <portSpacing port="sink_averagable 1" spacing="0"/>
             <portSpacing port="sink_averagable 2" spacing="0"/>
           </process>
         </operator>
         <connect from_port="input 1" to_op="Validation" to_port="training"/>
         <connect from_op="Validation" from_port="averagable 1" to_port="performance"/>
         <portSpacing port="source_input 1" spacing="0"/>
         <portSpacing port="source_input 2" spacing="0"/>
         <portSpacing port="sink_performance" spacing="0"/>
         <portSpacing port="sink_result 1" spacing="0"/>
       </process>
     </operator>
     <connect from_op="Read Excel" from_port="output" to_op="Optimize Parameters (Grid)" to_port="input 1"/>
     <connect from_op="Optimize Parameters (Grid)" from_port="performance" to_port="result 1"/>
     <connect from_op="Optimize Parameters (Grid)" from_port="parameter" to_port="result 2"/>
     <portSpacing port="source_input 1" spacing="0"/>
     <portSpacing port="sink_result 1" spacing="0"/>
     <portSpacing port="sink_result 2" spacing="0"/>
     <portSpacing port="sink_result 3" spacing="0"/>
   </process>
 </operator>
</process>
When I actually apply the received parameters I only get 52,05 %. (see code 2)
Code2:

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<process version="5.2.008">
 <context>
   <input/>
   <output/>
   <macros/>
 </context>
 <operator activated="true" class="process" compatibility="5.2.008" expanded="true" name="Process">
   <process expanded="true" height="467" width="748">
     <operator activated="true" class="read_excel" compatibility="5.2.008" expanded="true" height="60" name="Read Excel" width="90" x="45" y="30">
       <parameter key="excel_file" value="D:\PSY-DATA\06_HERZRATEN_PROJEKT\HR_KlassDaten.xlsx"/>
       <parameter key="imported_cell_range" value="A1:M4901"/>
       <parameter key="first_row_as_names" value="false"/>
       <list key="annotations">
         <parameter key="0" value="Name"/>
       </list>
       <list key="data_set_meta_data_information">
         <parameter key="0" value="Probandennummer.false.integer.attribute"/>
         <parameter key="1" value="Alter.false.integer.attribute"/>
         <parameter key="2" value="Altersgruppe.false.integer.attribute"/>
         <parameter key="3" value="Geschlecht.false.integer.attribute"/>
         <parameter key="4" value="Geschlechtsfaktor.false.integer.attribute"/>
         <parameter key="5" value="Mens.false.integer.attribute"/>
         <parameter key="6" value="RMSSD(ms).true.real.attribute"/>
         <parameter key="7" value="mean_RR(ms).true.real.attribute"/>
         <parameter key="8" value="std_RR(ms).true.real.attribute"/>
         <parameter key="9" value="mean_HR.true.real.attribute"/>
         <parameter key="10" value="std_HR.true.real.attribute"/>
         <parameter key="11" value="label.false.polynominal.attribute"/>
         <parameter key="12" value="label valenz.true.polynominal.label"/>
       </list>
     </operator>
     <operator activated="true" class="x_validation" compatibility="5.2.008" expanded="true" height="112" name="Validation" width="90" x="313" y="30">
       <parameter key="average_performances_only" value="false"/>
       <process expanded="true" height="511" width="365">
         <operator activated="true" class="support_vector_machine_libsvm" compatibility="5.2.008" expanded="true" height="76" name="SVM" width="90" x="137" y="30">
           <parameter key="gamma" value="3.0517578125E-5"/>
           <parameter key="C" value="0.03125"/>
           <parameter key="cache_size" value="250"/>
           <list key="class_weights"/>
           <parameter key="calculate_confidences" value="true"/>
         </operator>
         <connect from_port="training" to_op="SVM" to_port="training set"/>
         <connect from_op="SVM" from_port="model" to_port="model"/>
         <portSpacing port="source_training" spacing="0"/>
         <portSpacing port="sink_model" spacing="0"/>
         <portSpacing port="sink_through 1" spacing="0"/>
       </process>
       <process expanded="true" height="511" width="365">
         <operator activated="true" class="apply_model" compatibility="5.2.008" expanded="true" height="76" name="Apply Model" width="90" x="45" y="30">
           <list key="application_parameters"/>
         </operator>
         <operator activated="true" class="performance_classification" compatibility="5.2.008" expanded="true" height="76" name="Performance" width="90" x="205" y="30">
           <list key="class_weights"/>
         </operator>
         <connect from_port="model" to_op="Apply Model" to_port="model"/>
         <connect from_port="test set" to_op="Apply Model" to_port="unlabelled data"/>
         <connect from_op="Apply Model" from_port="labelled data" to_op="Performance" to_port="labelled data"/>
         <connect from_op="Performance" from_port="performance" to_port="averagable 1"/>
         <portSpacing port="source_model" spacing="0"/>
         <portSpacing port="source_test set" spacing="0"/>
         <portSpacing port="source_through 1" spacing="0"/>
         <portSpacing port="sink_averagable 1" spacing="0"/>
         <portSpacing port="sink_averagable 2" spacing="0"/>
       </process>
     </operator>
     <connect from_op="Read Excel" from_port="output" to_op="Validation" to_port="training"/>
     <connect from_op="Validation" from_port="averagable 1" to_port="result 1"/>
     <portSpacing port="source_input 1" spacing="0"/>
     <portSpacing port="sink_result 1" spacing="0"/>
     <portSpacing port="sink_result 2" spacing="0"/>
   </process>
 </operator>
</process>
[The codes above are only showing my setups for trouble-shooting.]

How is that possible?
Or am I doing something wrong? (by the way: my classification task is binary and my two classes are well balanced, (128 training vectors a 5 features for each class))

Thanks a lot in advance,
Sasch

Find more posts tagged with