mkallipo
/
affRo
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Update update_records.py

This commit is contained in:
Serafeim Chatzopoulos 2024-09-09 15:23:39 +02:00
parent eae8412a16
commit 78de49bb3a
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
update_records.py
View File

@ -2,15 +2,13 @@ import json
import os import os
from pyspark.sql import SparkSession from pyspark.sql import SparkSession
from affro_cluster import * from affro_cluster import *
import sys
folder_path = '/user/zeppelin/miriam.baglioni/AffStringFromIISDataset2' folder_path = sys.argv[1]
hdfs_output_path = '/tmp/affro/results' hdfs_output_path = sys.argv[2]
#folder_path = 'check'
json_file_names = []
# Initialize Spark session # Initialize Spark session
spark = SparkSession.builder.appName("JSONProcessing").getOrCreate() spark = SparkSession.builder.appName("AffRo").getOrCreate()
def remove_duplicates(list_of_dicts): def remove_duplicates(list_of_dicts):
# Use a set to store tuples of dictionary items to filter out duplicates # Use a set to store tuples of dictionary items to filter out duplicates