from flask import Flask, render_template, url_for, request, jsonify,send_file from flask_cors import CORS, cross_origin from werkzeug.utils import secure_filename from scipy import stats import sys import os import glob import re import csv import json import cdflib from astropy.utils.data import get_pkg_data_filename import magic from astropy.table import Table from astropy.io import fits import numpy as np from scipy.signal import find_peaks, peak_prominences import pandas as pd from statsmodels.tsa.seasonal import STL from scipy.integrate import simps from numpy import trapz from matplotlib import pyplot as plt app=Flask(__name__) cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' def reduce_noise_by_stl_trend(rate, time): window_width = 60 byteorder = rate.dtype.byteorder if byteorder=='=': byteorder=sys.byteorder print(byteorder) if byteorder == '<'or byteorder=='little': rate = (pd.Series(rate).rolling(window=window_width).mean().iloc[window_width-1:].values) time = (pd.Series(time).rolling(window=window_width).mean().iloc[window_width-1:].values) else: rate = (pd.Series(rate.byteswap().newbyteorder()).rolling(window=window_width).mean().iloc[window_width-1:].values) time = (pd.Series(time.byteswap().newbyteorder()).rolling(window=window_width).mean().iloc[window_width-1:].values) # fig = plt.figure(figsize=(18, 8)) return rate,time def find_peak(rate,time): x = rate peaks, _ = find_peaks(x) prominences, _, _ = peak_prominences(x, peaks) selected = prominences > 0.3 * (np.min(prominences) + np.max(prominences)) top = peaks[selected] topp = [] for i in range(0,len(top)): if x[top][i]>600: topp.append(top[i]) return topp def get_start_point(top,rate, time): x = rate i = top start = [] start_index = [] start_time = [] peak = [] peak_time = [] for i in top: print("peak coordinates : ",time[i],x[i]) peak.append(x[i]) peak_time.append(time[i]) while i>0: t = time[i] if((x[i]-x[i-1])<0.00001 and (x[i-1]-x[i-2])<0.00001 and (x[i-2]-x[i-3])<0.00001 and x[i]<0.5*(np.max(x)-np.min(x))): print("start coordinates : ",t,x[i]) start.append(x[i]) start_time.append(time[i]) start_index.append(i) break if( x[i-1]>=x[i] and x[i-2]>=x[i-1] and x[i-3]>=x[i-2] and x[i-3]>=1.005*x[i]): print("start coordinates : ",t,x[i]) start.append(x[i]) start_index.append(i) start_time.append(time[i]) break i = i-1 return start, start_index, start_time, peak , peak_time def get_end_time(top,start,rate,time): x = rate i = top end = [] end_index = [] end_time = [] for i in top: print("peak coordinates : ",time[i],x[i]) while i: t = time[i] if((x[i]-x[i+1])<0.00001 and (x[i+1]-x[i+2])<0.00001 and (x[i+2]-x[i+3])<0.00001 and x[i]<0.5*(np.max(x)-np.min(x))): print("end coordinates : ",t,x[i]) end.append(x[i]) end_index.append(i) end_time.append(time[i]) break if( x[i+1]>=x[i] and x[i+2]>=x[i+1] and x[i+3]>=x[i+2] and x[i+3]>=1.005*x[i]): print("end coordinates : ",t,x[i]) end.append(x[i]) end_index.append(i) end_time.append(time[i]) break i = i+1 if(i>=len(time)): break return end, end_index, end_time def area_under_curve(rate, start_index, end_index): area = [] for i in range(len(start_index)): y = rate[start_index[i]:end_index[i]+1] area.append(trapz(y, dx=1)) return area def flux_curve(df): df.fillna(0, inplace=True) window_width = 1 flux = (pd.Series(df['flux']).rolling(window=window_width).mean().iloc[window_width-1:].values) time = (pd.Series(df['time']).rolling(window=window_width).mean().iloc[window_width-1:].values) x = flux peaks, _ = find_peaks(x) prominences, _, _ = peak_prominences(x, peaks) selected = prominences > 0.3 * (np.min(prominences) + np.max(prominences)) top = peaks[selected] print(x[top]) x = flux i = top start = [] start_index = [] for i in top: print("peak coordinates : ",x[i]) while i>0: if((x[i]-x[i-1])<0.00001 and (x[i-1]-x[i-2])<0.00001 and (x[i-2]-x[i-3])<0.00001 and x[i]<0.1*(np.max(x)-np.min(x))): print("start coordinates : ",time[i],x[i]) start.append(x[i]) break if( x[i-1]>=x[i] and x[i-2]>=x[i-1] and x[i-3]>=x[i-2] and x[i-3]>=1.005*x[i]): print("start coordinates : ",time[i], x[i]) start.append(x[i]) start_index.append(i) break i = i-1 x = flux j = top end = [] end_index = [] for j in top: print("peak coordinates : ",time[j],x[j]) while j: t = time[j] try: if((x[j]-x[j+1])<0.00001 and (x[j+1]-x[j+2])<0.00001 and (x[j+2]-x[j+3])<0.00001 and x[j]<0.1*(np.max(x)-np.min(x))): print("end coordinates : ",t,x[j]) end.append(x[j]) break except: print("end coordinates : ",t,x[j-1]) end.append(x[j-1]) break if( x[j+1]>=x[j] and x[j+2]>=x[j+1] and x[j+3]>=x[j+2] and x[j+3]>=1.005*x[j]): print("end coordinates : ",t,x[j]) end.append(x[j]) end_index.append(j) break j = j+1 for s in range(len(start)): si = np.where(x == start[s])[0] ei = np.where(x == end[s])[0] try: x_f = np.delete(x, slice(si[0], ei[0]), 0) except: x_f=x flux_bc = np.mean(x_f, axis=0) return time[top],x[top], flux_bc def get_bc(start, end, rate): x = rate for s in range(len(start)): si = np.where(x == start[s])[0] ei = np.where(x == end[s])[0] try: x = np.delete(x, slice(si[0], ei[0]), 0) except: continue bc = np.mean(x, axis=0) return bc def classification_by_area(area): area_class = [] for i in range(0,len(area)): if(area[i]>=1e6): area_class.append("BRIGHT") elif(area[i]<1e6 and area[i]>=1e5): area_class.append("NORMAL") else: area_class.append("FAINT") return area_class def classification_by_duration(start_time,end_time): duration_class = [] for i in range(0,len(start_time)): duration = end_time[i]-start_time[i] if(duration<=3600): duration_class.append("SHORT DURATION OR IMPULSIVE EVENT") else: duration_class.append("LONG DURATION OR GRADUAL EVENT") return duration_class def append_to_dataframe(df,name,start,start_time,end,end_time,peak,peak_time,area,bc,area_class,duration_class): burst_time = [] rise_time = [] decay_time = [] for i in range(0,len(peak)): burst_time.append(end_time[i]-start_time[i]) rise_time.append(peak_time[i]-start_time[i]) decay_time.append(end_time[i]-peak_time[i]) dict = {'file_name':name,'start coordinate (x)':start_time, 'start coordinate (y)':start, 'peak coordinate (x)':peak_time, 'peak coordinate (y)':peak, 'end coordinate (x)':end_time, 'end coordinate (y)':end, 'total burst time':burst_time, 'rise time':rise_time, 'decay time':decay_time, 'area under curve':area,'background count Rate vs Time':bc, 'classfication by area':area_class, 'classification by duration':duration_class} df2 = pd.DataFrame(dict) df3 = pd.concat([df, df2], ignore_index = True) df3.reset_index() return df3 def classification_by_flux_peak(flux_peak): flux_class = [] for i in range(0,len(flux_peak)): if(flux_peak[i]<1e-7): flux_class.append('A') elif(flux_peak[i]>1e-7 and flux_peak[i]<1e-6): flux_class.append('B') elif(flux_peak[i]>1e-6 and flux_peak[i]<1e-5): flux_class.append('C') elif(flux_peak[i]>1e-5 and flux_peak[i]<1e-4): flux_class.append('M') else: flux_class.append('X') return flux_class def classification_by_flux_peak_by_bc(flux_peak,flux_bc): flux_class_bc = [] for i in range(0,len(flux_peak)): if((flux_peak[i]/flux_bc)<10): flux_class_bc.append('Type 1') elif((flux_peak[i]/flux_bc)>10 and (flux_peak[i]/flux_bc)<100): flux_class_bc.append('Type 2') else: flux_class_bc.append('Type 3') return flux_class_bc def append_to_dataframe(df,name,start,start_time,end,end_time,peak,peak_time,area,bc,area_class,duration_class): burst_time = [] rise_time = [] decay_time = [] for i in range(0,len(peak)): burst_time.append(end_time[i]-start_time[i]) rise_time.append(peak_time[i]-start_time[i]) decay_time.append(end_time[i]-peak_time[i]) dict = {'file_name':name,'start coordinate (x)':start_time, 'start coordinate (y)':start, 'peak coordinate (x)':peak_time, 'peak coordinate (y)':peak, 'end coordinate (x)':end_time, 'end coordinate (y)':end, 'total burst time':burst_time, 'rise time':rise_time, 'decay time':decay_time, 'area under curve':area,'background count Rate vs Time':bc, 'classfication by area':area_class, 'classification by duration':duration_class} df2 = pd.DataFrame(dict) df3 = pd.concat([df, df2], ignore_index = True) df3.reset_index() return df3 def flux_dataframe(df1,flux_file,flux_peak_time, flux_peak,flux_bc,flux_class,flux_class_bc): dict = {'flux_file_name':flux_file,'Peak Flux (x)':flux_peak_time,'Peak Flux (y)':flux_peak,'background count Flux vs Time':flux_bc,'Classification by Flux Peak':flux_class,'Classification by Flux Peak By Background Count':flux_class_bc} df2 = pd.DataFrame(dict) df3 = pd.concat([df1, df2], ignore_index = True) df3.reset_index() return df3 def store_data(zipname): name = zipname.split("_")[-2] year = name[:4] month = name[4:6] day = name[6:8] os.system("unzip "+zipname+" -d temp")#unzfileip os.system("rsync -av temp/xsm/data/ data")#merge os.system("rm -r temp")#temp removal os.system("rm -r "+zipname)#zip return year,month,day #data/year/month/day/calibrated/ch2_xsm_yeardaymonth_v1_level2.lc # fluxc.txt def path(year,month,day): lcpath = "data/"+year+"/"+month+"/"+day+"/calibrated/"+"ch2_xsm_"+year+month+day+"_v1_level2.lc" flux_path = "data/"+year+"/"+month+"/"+day+"/calibrated/fluxc.txt" return lcpath,flux_path def generate_flux(year,month,day): flag=0 os.system("xsmgenspec l1file=data/"+year+"/"+month+"/"+day+"/raw/ch2_xsm_"+year+month+day+"_v1_level1.fits specfile=data/"+year+"/"+month+"/"+day+"/calibrated/ch2_xsm_"+year+month+day+"_v1_flux.txt spectype=time-resolved tstart=0 tstop=0 tbinsize=1 hkfile=data/"+year+"/"+month+"/"+day+"/raw/ch2_xsm_"+year+month+day+"_v1_level1.hk safile=data/"+year+"/"+month+"/"+day+"/raw/ch2_xsm_"+year+month+day+"_v1_level1.sa gtifile=data/"+year+"/"+month+"/"+day+"/calibrated/ch2_xsm_"+year+month+day+"_v1_level2.gti") os.system("xsmcomputeflux data/"+year+"/"+month+"/"+day+"/calibrated/ch2_xsm_"+year+""+month+""+day+"_v1_flux.txt data/"+year+"/"+month+"/"+day+"/calibrated/fluxc.txt 1.5498 12.398") os.system("rm -r data/"+year+"/"+month+"/"+day+"/calibrated/ch2_xsm_"+year+""+month+""+day+"_v1_flux.txt") flag = os.system("rm -r data/"+year+"/"+month+"/"+day+"/calibrated/ch2_xsm_"+year+""+month+""+day+"_v1_flux.arf") if(flag!=0): os.system("touch data/"+year+"/"+month+"/"+day+"/calibrated/fluxc.txt") return 0 else: return 1 def choose1(x,y,maxsize): binsize = int(len(x)/maxsize) xnew = np.array([]) ynew = np.array([]) count = 0 while count < maxsize: count = int(count) xnew = np.append(xnew,x[count*binsize]) ynew = np.append(ynew,y[count*binsize]) count=count+1 return xnew,ynew def assign_status(df_rate,peak_time,start_time,end_time): df_rate['Status']='Normal' for i in range(0,len(df_rate)): if(df_rate.iloc[i,1] in peak_time): df_rate.iloc[i,3] = 'Peak' elif(df_rate.iloc[i,1] in start_time): df_rate.iloc[i,3] = 'Start' elif(df_rate.iloc[i,1] in end_time): df_rate.iloc[i,3] = 'End' else: df_rate.iloc[i,3] = 'Normal' return df_rate def make_json(csvFilePath, jsonFilePath): # create a dictionary data = {} # Open a csv reader called DictReader with open(csvFilePath, encoding='utf-8') as csvf: csvReader = csv.DictReader(csvf) # Convert each row into a dictionary # and add it to data for rows in csvReader: # Assuming a column named 'No' to # be the primary key key = rows['No'] data[key] = rows # Open a json writer, and use the json.dumps() # function to dump data with open(jsonFilePath, 'w', encoding='utf-8') as jsonf: jsonf.write(json.dumps(data, indent=4)) @app.route('/', methods=['GET']) @cross_origin() def home(): return "

Hi from Backend!

" @app.route('/api/upload', methods=['POST']) @cross_origin() def upload(): # Get the file from post request f = request.files['imgfile'] f.save(secure_filename(f.filename)) file_path = f.filename if(file_path[-4:]!=".zip"): lcpath = f.filename is_flux=0 os.system("touch tempflux.csv") flux_path = "tempflux.csv" else: year, month, day = store_data(file_path) is_flux = generate_flux(year, month, day) lcpath, flux_path = path(year, month, day) df = pd.DataFrame(columns=['file_name', 'start coordinate (x)', 'start coordinate (y)', 'peak coordinate (x)', 'peak coordinate (y)', 'end coordinate (x)', 'end coordinate (y)', 'total burst time', 'rise time', 'decay time', 'area under curve', 'background count Rate vs Time', 'classfication by area', 'classification by duration']) flux_df = pd.DataFrame(columns=['flux_file_name', 'Peak Flux (x)', 'Peak Flux (y)', 'background count Flux vs Time', 'Classification by Flux Peak', 'Classification by Flux Peak By Background Count']) # df1 = pd.read_table(flux_path, delimiter=' ', header=None) filetype = magic.from_file(lcpath) print("\n\n\n"+filetype+"\n\n\n") if 'ASCII' in filetype: df_x = pd.read_csv(lcpath, sep=" ", skipinitialspace=True) df_x.columns = ["time", "rate"] rate, time = reduce_noise_by_stl_trend(np.array(df_x["rate"], dtype=float), np.array(df_x["time"], dtype=float)) # print(df_x) # df_x.to_excel("excel_try.xlsx",index=False) elif 'CSV' in filetype: df_x = pd.read_csv(lcpath) df_x.columns = ["time", "rate"] rate, time = reduce_noise_by_stl_trend(np.array(df_x["rate"], dtype=float), np.array(df_x["time"], dtype=float)) elif 'Excel' in filetype: df_x = pd.read_excel(lcpath) df_x.columns = ["time", "rate"] # plt.plot(df_x['time'], df_x['rate']) # print(df_x) rate = np.array(df_x['rate']) time = np.array(df_x['time']) # plt.plot(time,rate) rate, time = reduce_noise_by_stl_trend(rate, time) elif 'FITS' in filetype: image_file = fits.open(lcpath) file_data = image_file[1].data rate, time = reduce_noise_by_stl_trend(file_data["rate"], file_data["time"]) elif 'FPT' in filetype or 'data' in filetype: cdf_file = cdflib.CDF(lcpath) arr = np.array((cdf_file.varget(variable='Sample Light Curve')[0])) rate = np.array([element[1] for element in arr]) time = np.array([element[0] for element in arr]) rate, time = reduce_noise_by_stl_trend(rate, time) # df_flux.to_csv(flux_path+'.csv', index = None) # image_file = fits.open(lcpath) # file_data = image_file[1].data #rate, time = reduce_noise_by_stl_trend(file_data) # rate_time_array = np.transpose(np.array([time,rate])) # time2,rate2 = time,rate if(len(time)>=1000): time,rate=choose1(time,rate,1000) df_rate = pd.DataFrame({ 'time':time, 'rate':rate}, index=None) # df_rate.to_csv(path+file_name+'.csv', index=None, header=False) top = find_peak(rate,time) start, start_index, start_time,peak,peak_time = get_start_point(top,rate,time) end, end_index,end_time = get_end_time(top,start,rate,time) print(start) print(end) print(rate) area = area_under_curve(rate, start_index, end_index) bc = get_bc(start, end, rate) area_class = classification_by_area(area) duration_class = classification_by_duration(start_time,end_time) df = append_to_dataframe(df,flux_path,start,start_time,end,end_time,peak,peak_time,area,bc,area_class,duration_class) df_rate['status'] ='Normal' #df_rate = assign_status(df_rate,peak_time,start_time,end_time) df_rate['status'] = df_rate['time'].apply(lambda x: 'Peak' if x in peak_time else('Start' if x in start_time else('End' if x in end_time else 'Normal'))) #df_flux = assign_status(df_flux,flux_peak_time,[],[]) if is_flux: df_flux = pd.read_csv(flux_path, delimiter = ' ',usecols = [2]) df_flux.columns = ['flux'] df_flux['time'] = df_flux.index df_flux = df_flux[['time', 'flux']] tm,rt = choose1(df_flux['time'],df_flux['flux'],1000) df_temp=pd.DataFrame() df_temp['time']=tm df_temp['flux'] =rt df_flux=df_temp flux_peak_time, flux_peak, flux_bc = flux_curve(df_flux) flux_class = classification_by_flux_peak(flux_peak) flux_class_bc = classification_by_flux_peak_by_bc(flux_peak,flux_bc) flux_df = flux_dataframe(flux_df,lcpath,flux_peak_time, flux_peak, flux_bc,flux_class,flux_class_bc) df_flux['status'] ='Normal' df_flux['status'] = df_flux['time'].apply(lambda x: 'Peak' if x in flux_peak_time else 'Normal') else: df_flux = pd.DataFrame(columns = ['flux_file_name','Peak Flux (x)','Peak Flux (y)','background count Flux vs Time','Classification by Flux Peak','Classification by Flux Peak By Background Count']) try: # lc_orig_df = pd.read_csv("CSV/lc.csv") # flux_orig_df = pd.read_csv("CSV/flux.csv") # all_lc_orig_df = pd.read_csv("CSV/all_lc.csv") # all_flux_orig_df = pd.read_csv("CSV/all_flux.csv") # pd.concat([lc_orig_df, df], ignore_index = True).to_csv("CSV/lc.csv", index=False) # pd.concat([flux_orig_df, flux_df], ignore_index = True).to_csv("CSV/flux.csv", index=False) # pd.concat([all_lc_orig_df, df_rate], ignore_index = True).to_csv("CSV/all_lc.csv", index=False) # pd.concat([all_flux_orig_df, df_flux], ignore_index = True).to_csv("CSV/all_flux.csv", index=False) df.to_csv(f'./CSV/lc.csv') df_rate.to_csv(f'./CSV/all_lc.csv') if is_flux: flux_df.to_csv(f'./CSV/flux.csv') df_flux.to_csv(f'./CSV/all_flux.csv') except: df.to_csv(f'./CSV/lc.csv') flux_df.to_csv(f'./CSV/flux.csv') df_rate.to_csv(f'./CSV/all_lc.csv') df_flux.to_csv(f'./CSV/all_flux.csv') return jsonify({'status': 'ok'}) @app.route('/api/data/lcfull', methods=['GET']) @cross_origin() def lcfulldata(): try: lc_csv = pd.read_csv(r'CSV/all_lc.csv') lc_csv.to_json(r'JSON/all_lc.json') with open('JSON/all_lc.json', 'r') as file: lcJSON = file.read() return jsonify(lcJSON) except: return "No File Provided" @app.route('/api/data/fluxfull', methods=['GET']) @cross_origin() def fluxfulldata(): try: flux_csv = pd.read_csv(r'CSV/all_flux.csv') flux_csv.to_json(r'JSON/all_flux.json') with open('JSON/all_flux.json', 'r') as file: fluxJSON = file.read() return jsonify(fluxJSON) except: return "No File Provided" @app.route('/api/data/lc', methods=['GET']) @cross_origin() def lcData(): try: lc_csv = pd.read_csv(r'CSV/lc.csv') lc_csv.columns = lc_csv.columns.str.replace(' ','_') lc_csv.columns = lc_csv.columns.str.replace('(','_') lc_csv.columns = lc_csv.columns.str.replace(')','_') lc_csv.to_json(r'JSON/lc.json') with open('JSON/lc.json', 'r') as file: lcJSON = file.read() return jsonify(lcJSON) except: return "No File Provided" @app.route('/api/data/flux', methods=['GET']) @cross_origin() def fluxData(): try: flux_csv = pd.read_csv(r'CSV/flux.csv') flux_csv.columns = flux_csv.columns.str.replace(' ','_') flux_csv.columns = flux_csv.columns.str.replace('(','_') flux_csv.columns = flux_csv.columns.str.replace(')','_') flux_csv.to_json(r'JSON/flux.json') with open('JSON/flux.json', 'r') as file: fluxJSON = file.read() return jsonify(fluxJSON) except: return "No File Provided" if __name__ == "__main__": app.run(debug=False, host='0.0.0.0', port=8080) # debug was initially True