feat: implement logistic regression pipeline with automated JASP-formatted report generation and data inspection utilities

Project_Code_and_Results/4_Logistic_Regression/Logistic_Results_Decision_Final.csv

@@ -1,25 +1,25 @@
-,Beta (B),P-value,Odds Ratio EXP(B),Significance
-const,-13.3839,0.0078,0.0,**
-Frequency,0.7183,0.0098,2.051,**
-Career_6,4.1539,0.0105,63.6799,*
-Rooftop,1.7017,0.0123,5.4833,*
-Income,0.0045,0.0229,1.0045,*
-Transportation_4,4.6473,0.0284,104.3069,*
-MEAN CES,1.8488,0.0288,6.3523,*
-Career_4,2.8862,0.0334,17.9242,*
-Career_3,3.4267,0.0368,30.7762,*
-Garden,-1.4026,0.0446,0.246,*
-MEAN DES,-0.4982,0.0586,0.6076,
-Transportation_3,2.1343,0.0739,8.4512,
-Literacy,0.5519,0.0906,1.7365,
-Gender_2,1.1137,0.1406,3.0455,
-Nature,-0.6135,0.2466,0.5415,
-MEAN RES,-0.6547,0.2723,0.5196,
-Residential,0.6836,0.391,1.981,
-Time,-0.1722,0.5721,0.8418,
-Distance,-0.161,0.6202,0.8513,
-Transportation_2,0.6629,0.6341,1.9404,
-Recreation,-0.1187,0.7927,0.8881,
-Career_2,0.347,0.8054,1.4148,
-Park,-0.1024,0.8656,0.9027,
-Agriculture,-0.0665,0.8894,0.9356,
+,Beta (B),S.E.,P-value,Odds Ratio EXP(B),Significance
+const,-13.3839,5.0267,0.0078,0.0,**
+Frequency,0.7183,0.278,0.0098,2.051,**
+Career_6,4.1539,1.6229,0.0105,63.6799,*
+Rooftop,1.7017,0.6798,0.0123,5.4833,*
+Income,0.0045,0.002,0.0229,1.0045,*
+Transportation_4,4.6473,2.1208,0.0284,104.3069,*
+MEAN CES,1.8488,0.8458,0.0288,6.3523,*
+Career_4,2.8862,1.3569,0.0334,17.9242,*
+Career_3,3.4267,1.6413,0.0368,30.7762,*
+Garden,-1.4026,0.6984,0.0446,0.246,*
+MEAN DES,-0.4982,0.2635,0.0586,0.6076,
+Transportation_3,2.1343,1.1942,0.0739,8.4512,
+Literacy,0.5519,0.3261,0.0906,1.7365,
+Gender_2,1.1137,0.7557,0.1406,3.0455,
+Nature,-0.6135,0.5294,0.2466,0.5415,
+MEAN RES,-0.6547,0.5964,0.2723,0.5196,
+Residential,0.6836,0.7969,0.391,1.981,
+Time,-0.1722,0.3048,0.5721,0.8418,
+Distance,-0.161,0.3249,0.6202,0.8513,
+Transportation_2,0.6629,1.3928,0.6341,1.9404,
+Recreation,-0.1187,0.4515,0.7927,0.8881,
+Career_2,0.347,1.4082,0.8054,1.4148,
+Park,-0.1024,0.6051,0.8656,0.9027,
+Agriculture,-0.0665,0.4784,0.8894,0.9356,

Project_Code_and_Results/4_Logistic_Regression/export_jasp_layout.py

@@ -0,0 +1,75 @@
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+import pandas as pd
+import numpy as np
+import os
+
+base_dir = r'c:\Users\NASPC\Documents\Du án tại SG tháng 8\Project_Code_and_Results\4_Logistic_Regression'
+
+file_donation = os.path.join(base_dir, 'Logistic_Results_Donation_Firth.csv')
+file_decision = os.path.join(base_dir, 'Logistic_Results_Decision_Final.csv')
+
+def process_results(filepath, model_name="M₁"):
+    df = pd.read_csv(filepath, index_col=0)
+    
+    # Check if we have S.E. or std err
+    se_col = 'S.E.' if 'S.E.' in df.columns else ('std err' if 'std err' in df.columns else None)
+    if not se_col:
+        # For some files it might be 'SE'
+        if 'SE' in df.columns: se_col = 'SE'
+    
+    # Recreate necessary metrics for JASP format
+    estimate = df['Beta (B)']
+    se = df[se_col] if se_col else 0
+    odds_ratio = df['Odds Ratio EXP(B)']
+    p_val = df['P-value']
+    
+    # Calculate Z and Wald
+    z_stat = estimate / se
+    wald = z_stat ** 2
+    lower = np.exp(estimate - 1.96 * se)
+    upper = np.exp(estimate + 1.96 * se)
+    
+    # Rename index to match JASP
+    # JASP formats: e.g. "Gender_2" -> "Gender (2)", "const" -> "(Intercept)"
+    def map_name(name):
+        if name == 'const': return '(Intercept)'
+        if '_' in name:
+            parts = name.split('_')
+            return f"{parts[0]} ({parts[1]})"
+        return name
+    
+    new_index = [map_name(str(i)) for i in df.index]
+    
+    # Build the target dataframe
+    out_df = pd.DataFrame({
+        'Model': [model_name] + [np.nan] * (len(df) - 1),
+        '': new_index,
+        'Estimate': estimate.values,
+        'Standard Error': se.values if se_col else [np.nan]*len(df),
+        'Odds Ratio': odds_ratio.values,
+        'z': z_stat.values,
+        'Wald Statistic': wald.values,
+        'df': [1] * len(df),
+        'p': p_val.values,
+        'Lower bound': lower.values,
+        'Upper bound': upper.values
+    })
+    
+    # Format p-values < .00001
+    out_df['p'] = out_df['p'].apply(lambda x: '< .00001' if pd.notnull(x) and x < 0.00001 else (round(x, 5) if pd.notnull(x) else x))
+    out_df = out_df.round(5)
+    return out_df
+
+out_path = r'c:\Users\NASPC\Documents\Du án tại SG tháng 8\Project_Code_and_Results\4_Logistic_Regression\KẾT_QUẢ_FINAL_JASP_LAYOUT.xlsx'
+
+with pd.ExcelWriter(out_path, engine='openpyxl') as writer:
+    if os.path.exists(file_donation):
+        df_don = process_results(file_donation, "M₁")
+        df_don.to_excel(writer, sheet_name='Donation_New', index=False)
+    
+    if os.path.exists(file_decision):
+        df_dec = process_results(file_decision, "M₁")
+        df_dec.to_excel(writer, sheet_name='Decision_New', index=False)
+
+print(f"Exported successfully to {out_path}")

Project_Code_and_Results/4_Logistic_Regression/run_final_logistic.py

@@ -69,6 +69,7 @@ model_dec = sm.Logit(y_dec, X)
 res_dec = model_dec.fit(method='newton', maxiter=1000, disp=False)
 sum_dec = pd.DataFrame({
     'Beta (B)': res_dec.params,
+    'S.E.': res_dec.bse,
     'P-value': res_dec.pvalues,
     'Odds Ratio EXP(B)': np.exp(res_dec.params)
 }).round(4)

Project_Code_and_Results/Utils/create_master_excel.py

@@ -0,0 +1,102 @@
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+import pandas as pd
+import numpy as np
+import openpyxl
+from openpyxl.utils.dataframe import dataframe_to_rows
+from openpyxl.styles import Font, Alignment
+import os
+import shutil
+
+base_dir = r'c:\Users\NASPC\Documents\Du án tại SG tháng 8'
+orig_file = os.path.join(base_dir, 'KẾT QUẢ PHÂN TÍCH.xlsx')
+final_file = os.path.join(base_dir, 'Project_Code_and_Results', 'KẾT_QUẢ_PHÂN_TÍCH_MASTER.xlsx')
+
+# Copy original to new file
+shutil.copy(orig_file, final_file)
+
+# Load workbook
+wb = openpyxl.load_workbook(final_file)
+
+# Helper to process CSV into JASP layout dataframe
+def get_jasp_df(filepath, model_name="M₁"):
+    df = pd.read_csv(filepath, index_col=0)
+    se_col = 'S.E.' if 'S.E.' in df.columns else ('std err' if 'std err' in df.columns else 'SE')
+    
+    estimate = df['Beta (B)']
+    se = df[se_col]
+    odds_ratio = df['Odds Ratio EXP(B)']
+    p_val = df['P-value']
+    
+    z_stat = estimate / se
+    wald = z_stat ** 2
+    lower = np.exp(estimate - 1.96 * se)
+    upper = np.exp(estimate + 1.96 * se)
+    
+    def map_name(name):
+        if name == 'const': return '(Intercept)'
+        if '_' in name:
+            parts = name.split('_')
+            return f"{parts[0]} ({parts[1]})"
+        return name
+    
+    new_index = [map_name(str(i)) for i in df.index]
+    
+    out_df = pd.DataFrame({
+        'Model': [model_name] + [np.nan] * (len(df) - 1),
+        '': new_index,
+        'Estimate': estimate.values,
+        'Standard Error': se.values,
+        'Odds Ratio': odds_ratio.values,
+        'z': z_stat.values,
+        'Wald Statistic': wald.values,
+        'df': [1] * len(df),
+        'p': p_val.values,
+        'Lower bound': lower.values,
+        'Upper bound': upper.values
+    })
+    
+    # Format
+    out_df['p'] = out_df['p'].apply(lambda x: '< .00001' if pd.notnull(x) and x < 0.00001 else (round(x, 5) if pd.notnull(x) else x))
+    out_df = out_df.round(5)
+    return out_df
+
+# Replace a sheet's content
+def replace_sheet(sheet_name, csv_path):
+    idx = wb.sheetnames.index(sheet_name)
+    del wb[sheet_name]
+    ws = wb.create_sheet(sheet_name, idx)
+            
+    # Write Title
+    ws.cell(row=1, column=1, value="Logistic Regression (Updated with Firth/Optimized)").font = Font(bold=True)
+    ws.cell(row=3, column=1, value="Coefficients").font = Font(bold=True)
+    
+    # Headers
+    df_jasp = get_jasp_df(csv_path)
+    headers = list(df_jasp.columns)
+    
+    for c_idx, col_name in enumerate(headers, 1):
+        cell = ws.cell(row=4, column=c_idx, value=col_name)
+        cell.font = Font(bold=True)
+        cell.alignment = Alignment(horizontal='center')
+        
+    ws.cell(row=3, column=10, value="95% Confidence interval").font = Font(bold=True)
+    ws.cell(row=3, column=10).alignment = Alignment(horizontal='center')
+    
+    # Write Data
+    for r_idx, row in enumerate(dataframe_to_rows(df_jasp, index=False, header=False), 5):
+        for c_idx, value in enumerate(row, 1):
+            # Check for NaN float
+            if isinstance(value, float) and np.isnan(value):
+                ws.cell(row=r_idx, column=c_idx, value="")
+            else:
+                ws.cell(row=r_idx, column=c_idx, value=value)
+
+path_don = os.path.join(base_dir, 'Project_Code_and_Results', '4_Logistic_Regression', 'Logistic_Results_Donation_Firth.csv')
+path_dec = os.path.join(base_dir, 'Project_Code_and_Results', '4_Logistic_Regression', 'Logistic_Results_Decision_Final.csv')
+
+replace_sheet('Donation', path_don)
+replace_sheet('Decision', path_dec)
+
+wb.save(final_file)
+print("Tạo thành công MASTER file!")

Project_Code_and_Results/Utils/inspect_ket_qua.py

@@ -0,0 +1,22 @@
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+import pandas as pd
+import os
+
+file_path = r'c:\Users\NASPC\Documents\Du án tại SG tháng 8\KẾT QUẢ PHÂN TÍCH.xlsx'
+
+if not os.path.exists(file_path):
+    print("File not found")
+    sys.exit()
+
+print(f"Reading file: {file_path}")
+xl = pd.ExcelFile(file_path)
+print(f"Sheet names: {xl.sheet_names}\n")
+
+for sheet in xl.sheet_names:
+    print(f"--- Sheet: {sheet} ---")
+    df = xl.parse(sheet)
+    print(f"Shape: {df.shape}")
+    print(f"Columns: {list(df.columns)}")
+    print(f"First few rows:\n{df.head(3)}\n")
+

Project_Code_and_Results/Utils/inspect_layout.py

@@ -0,0 +1,11 @@
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+import pandas as pd
+
+file_path = r'c:\Users\NASPC\Documents\Du án tại SG tháng 8\KẾT QUẢ PHÂN TÍCH.xlsx'
+df_donation = pd.read_excel(file_path, sheet_name='Donation')
+df_decision = pd.read_excel(file_path, sheet_name='Decision')
+
+print("--- Donation Sheet Layout ---")
+print(df_donation.head(20).to_string())
+