-- ==================================================================
-- 20DayAuto.scpt - Stephen Randall - 2012
-- --------------------------------------------------
-- AppleScript that automatically downloads new files from APL, runs calculations, makes plots, and
-- updates the Cassini server.
-- Maintenence/Debugging help:
-- + Make sure that you're not using Mail.app on this computer. If you need/want to, you
-- will need to edit the email warning code.
-- + The script is dependent on file structure. If the script needs to move to
-- another computer, a number of changes will be needed.
-- + File names need to stay relatively consistent. If they ever go to v38, code will be needed
-- to account for that.
-- + In the case of a crash, the event log should be of some help. In addition, the program should
-- be able to perform a clean run the next time it's called by doing some error checking and
-- correcting before re-running.
-- + The name of the temporary directory is found in a .txt file in local_chems_files. There is an un-enforced
-- agreement between this script and the IDL routine that they should both use that directory. Breaking
-- that agreement will cause the script to perform in unpredictable ways.
-- + Avoid moving and changing items in the local_chems_files directory.
-- + I assume that APL will release [but not necessarily update] the PHA and Basic Rates data at the same
-- time. If this is not the case, it's possible for the IDL routine to be called without the necessary data.
-- + The file-name parsing towards the end is dependent on the default naming convention of the IDL
-- routines. Please don't change those.
-- + It is assumed that the location of XQuartz.app is /Applications/Utilities.
-- + In addition, the final file moves and clean-up is dependent on the current system set-up. If you move
-- the folders where the output is stored, the clean-up code will have to be modified accordingly.
-- + The ephem files are listed in defaults.txt. This is separate from the file names listed in the phaflux and
-- phaplot default files.
-- + There exists a "kill switch" when running the IDL routines. This is in place to prevent IDL from getting
-- stuck in a loop when it's reading a data file without all the correct information. While the script is
-- supposed to clean up after itself in this case, its behavior is technically undefined.
-- ==================================================================
global emailSubject
global emailContent
global periodoffset
-- initialize global email related variables
set emailSubject to "No files downloaded from APL today"
set emailContent to "No new files found"
-- Logging function
on log_text(msg)
set theLine to (do shell script "date +'%Y-%m-%d %H:%M:%S'" as string) & " " & msg
do shell script "echo " & quoted form of theLine & " >> ~/local_chems_files/20DayAuto_events.log"
end log_text
-- ==================================================================
-- Get the latest files from the APL server
-- ==================================================================
do shell script "/usr/local/bin/python3 ~/local_chems_files/getCHEMSFiles.py > ~/local_chems_files/pyout.txt"
-- If we couldn't connect to the APL servers, then send an email
try
set pyout to read (file "Macintosh HD:Users:stephen:local_chems_files:pyout.txt")
set pylines to count paragraphs in pyout
if pylines is greater than 3 then
my log_text("rsync failed to connect, email sent")
set theSubject to "Script Notification: rsync Connect Failed"
set dateToday to date string of (current date)
set theContent to "The Cassini/CHEMS script ran this morning (" & dateToday & "), but rsync failed to connect to the APL servers."
do shell script "cat << EOF | /usr/sbin/sendmail -t
From: CASSINI Mac
Subject: " & theSubject & "
Date: " & dateToday & "
To: Douglas C. Hamilton
CC: Scott
" & theContent & "
EOF"
-- tell application "Mail"
-- Create the message
-- set theMessage to make new outgoing message with properties {subject:theSubject, content:theContent, visible:false}
-- Set the recipients
-- tell theMessage
-- make new to recipient with properties {name:"Doug Hamilton", address:"dch@umd.edu"}
-- make new to recipient with properties {name:"Scott Lasley", address:"slasley@space.umd.edu"}
-- send
-- end tell
-- delay 20 -- let the message send
-- quit
-- end tell
return
end if
on error
-- Do nothing; this is the normal behavior
end try
-- ==================================================================
-- Create a temporary directory to work in; set up the log file
-- ==================================================================
do shell script "rm -f ~/local_chems_files/20DayAuto_events.log"
set tf to (open for access (file "Users:stephen:local_chems_files:temp_dir.txt"))
set temp_dir to (read tf for (get eof tf))
if fileExists(temp_dir) then do shell script "rm -r " & quoted form of temp_dir
do shell script "mkdir " & quoted form of temp_dir
-- ==================================================================
-- Figure out which periods to update
-- Periods must include at least one file from newFiles.txt [both pha and sci] and must have all files necessary
-- in the 20-day period
-- ==================================================================
try
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:newTrange.txt") with write permission
on error number error_number
my log_text(error_number)
close access (file "Macintosh HD:Users:stephen:local_chems_files:newTrange.txt")
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:newTrange.txt") with write permission
end try
set eof of write_file to 0
write "Time Range (yyyy ddd hhmm)" to write_file
set periods to {}
set pcount to 1
set the end of periods to pcount
set AppleScript's text item delimiters to "/"
try
set new_files to 1
set newFilesList to (read file "Macintosh HD:Users:stephen:local_chems_files:newFiles.txt" as {text})
set emailSubject to ((count paragraphs of newFilesList) - 1 as text) & " files downloaded from APL"
set emailContent to "Downloaded and processed these files from APL:
" & newFilesList as text
-- repeat with file_path in paragraphs of (read file "Macintosh HD:Users:stephen:local_chems_files:newFiles.txt" as {text})
repeat with file_path in paragraphs of newFilesList
my log_text("file_path = " & file_path)
if length of file_path is greater than 0 then
set file_name to (text item 6 of file_path)
set doy to (text 15 through 17 of file_name) as integer
set y to (text 11 through 14 of file_name) as integer
set ud to ((doy div 20) as integer) * 20 + 20
set ld to (ud - 20)
if not (periods contains ((y as text) & formatDay(ud))) then
set item pcount of periods to ((y as text) & formatDay(ud))
set pcount to 1 + pcount
set the end of periods to pcount
my log_text("tryNewPeriod(" & file_path & ", " & file_name & ", " & y & ", " & ud & ", " & ld & ", " & write_file & ")")
tryNewPeriod(file_path, file_name, y, ud, ld, write_file)
-- Take care of values that need to trigger multiple periods
if (y - 1) mod 4 is equal to 0 and doy ≤ 4 and not (periods contains ((y as text) & formatDay(4))) then
my log_text("tryNewPeriod(" & file_path & ", " & file_name & ", " & y & ", " & 4 & ", " & 350 & ", " & write_file & ")")
tryNewPeriod(file_path, file_name, y, 4, 350, write_file)
set item pcount of periods to ((y as text) & formatDay(4))
set pcount to 1 + pcount
set the end of periods to pcount
else if doy ≤ 5 and not (periods contains ((y as text) & formatDay(5))) then
my log_text("tryNewPeriod(" & file_path & ", " & file_name & ", " & y & ", " & 5 & ", " & 350 & ", " & write_file & ")")
tryNewPeriod(file_path, file_name, y, 5, 350, write_file)
set item pcount of periods to ((y as text) & formatDay(5))
set pcount to 1 + pcount
set the end of periods to pcount
else if doy ≥ 350 and not (periods contains ((y as text) & formatDay(380))) then
my log_text("tryNewPeriod(" & file_path & ", " & file_name & ", " & y & ", " & 380 & ", " & 0 & ", " & write_file & ")")
tryNewPeriod(file_path, file_name, y, 380, 0, write_file)
set item pcount of periods to ((y as text) & formatDay(380))
set pcount to 1 + pcount
set the end of periods to pcount
end if
end if
end if
end repeat
on error msg number num
set new_files to 0
my log_text("No new files")
end try
repeat with p in periods
set txt to "period info: "
my log_text(txt & p)
end repeat
close access write_file
-- Attempt to add the new period to the list
on tryNewPeriod(file_path, file_name, y, ud, ld, write_file)
-- Set special cases
set yoff to 0
if ud is equal to 20 then set ld to 1
if ud is equal to 380 then -- 360 ≤ doy
set ld to 350
if y mod 4 is equal to 0 then
set ud to 4
else
set ud to 5
end if
set yoff to 1
end if
-- Test if the necessary file is there
set fn to necessaryFile((text items 4 through 5 of file_path) as text, file_name, ud, y + yoff)
if ud ≤ 5 and yoff is equal to 0 then set y to y - 1
-- 5/11/15 remove check for fileExists to create plots every day (hopefully)
-- if fileExists(fn) then
-- writeFile(write_file, ld, ud, y)
-- end if
writeFile(write_file, ld, ud, y)
if fileExists(fn) then
set periodoffset to 0
else
set periodoffset to 20
end if
end tryNewPeriod
-- Return the name of the necessary file for the period
on necessaryFile(dir, file_name, ud, y)
return ("/Users/stephen/" & dir & "/" & (text 1 through 10 of file_name) & y & formatDay(ud) & (text 18 through end of file_name))
end necessaryFile
-- Write the file
on writeFile(write_file, ld, ud, y)
write formatPeriod(ld, ud, y) to write_file
end writeFile
-- Tests a file's existence
on fileExists(posixPath)
set ex to ((do shell script "if test -e " & quoted form of posixPath & "; then echo 1; else echo 0; fi") as integer) as boolean
if ex then
my log_text(posixPath & " exists")
else
my log_text(posixPath & " doesn't exist")
end if
return ex
end fileExists
-- Formats an integer to three digits
on formatDay(day_num)
set z to ""
if day_num < 100 then set z to "0"
if day_num < 10 then set z to "00"
return z & day_num
end formatDay
-- Formats the entire period
on formatPeriod(ld, ud, y)
set yt to y
if ld > ud then set yt to y + 1
return & y & " " & formatDay(ld) & " 0000-" & yt & " " & formatDay(ud) & " 0000"
end formatPeriod
-- ==================================================================
-- Use those periods to update the flux file listing and xdr file listing
-- ==================================================================
try
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:newFluxList.txt") with write permission
on error number error_number
my log_text(error_number)
close access (file "Macintosh HD:Users:stephen:local_chems_files:newFluxList.txt")
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:newFluxList.txt") with write permission
end try
set eof of write_file to 0
set AppleScript's text item delimiters to " "
set counter to 0
try
repeat with period in paragraphs of (read file "Macintosh HD:Users:stephen:local_chems_files:newTrange.txt" as {text})
if counter is not equal to 0 then
if counter is equal to 2 then
set newline to return
else if counter is equal to 1 then
set newline to ""
set counter to counter + 1
end if
write newline & temp_dir & "/" & (text item 1 of period) & (text item 2 of period) & "_0000-" & (text item 4 of period) & "_0000_00060Flux_incr.txt" to write_file
else
set counter to counter + 1
end if
end repeat
on error msg number num
my log_text((num as string) & " - " & msg)
end try
close access write_file
try
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:newxdrList.txt") with write permission
on error number error_number
my log_text(error_number)
close access (file "Macintosh HD:Users:stephen:local_chems_files:newxdrList.txt")
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:newxdrList.txt") with write permission
end try
set eof of write_file to 0
set AppleScript's text item delimiters to " "
set counter to 0
try
repeat with period in paragraphs of (read file "Macintosh HD:Users:stephen:local_chems_files:newTrange.txt" as {text})
if counter is not equal to 0 then
if counter is equal to 2 then
set newline to return
else if counter is equal to 1 then
set newline to ""
set counter to counter + 1
end if
set filepre to temp_dir & "/" & (text item 1 of period) & (text item 2 of period) & (text item 3 of period) & (text item 4 of period) & "0000_"
write newline & filepre & "00-07PHAhist.xdr" to write_file
if newline is equal to "" then set newline to return
write newline & filepre & "08-15PHAhist.xdr" to write_file
write newline & filepre & "16-23PHAhist.xdr" to write_file
write newline & filepre & "24-31PHAhist.xdr" to write_file
write newline & filepre & "00-31PHAhist.xdr" to write_file
else
set counter to counter + 1
end if
end repeat
on error msg number num
my log_text((num as string) & " - " & msg)
end try
close access write_file
-- ==================================================================
-- Run the IDL routines to calculate flux and make plots [Incomplete; need to read in periods and tempdir]
-- phaflux.pro needs to be able to iterate through the period array and take the custom flux file list
-- Note that IDL needs X11 to be running
-- ==================================================================
-- set msg to "Opening Xquartz"
-- do shell script "/usr/bin/open /Applications/Utilities/XQuartz.app"
set msg to "IDL starting"
my log_text(msg)
-- Sometimes, APL will release incomplete data files. In that case, IDL will continue running forever. To counter this,
-- the IDL routine is confined to 3 hours; any longer and the script is killed.
set msg to "IDL time out"
set idl_complete to 0
with timeout of 10800 seconds
do shell script "export IDL_DIR=/Applications/exelis/idl; source $IDL_DIR/bin/idl_setup.bash; /usr/local/bin/idl < ~/IDL/Master_Program/IDLCalls.pro >> ~/local_chems_files/20DayAuto_events.log"
set idl_complete to 1
set msg to "IDL routine successful"
end timeout
my log_text(msg)
if idl_complete is equal to 1 then
-- Refresh the Finder
do shell script "killall Finder"
delay 12
-- ==================================================================
-- Hit the created files with a pstopdf and then delete the .ps files
-- ==================================================================
do shell script "find " & quoted form of temp_dir & " -name \"*ps\" -exec pstopdf {} \\;"
do shell script "find " & quoted form of temp_dir & " -name \"*ps\" -exec rm {} \\;"
set msg to "Converted to pdfs; removed all postscripts"
my log_text(msg)
delay 8
-- ==================================================================
-- Put the new files up on the server [cURL] and move the files [flux and output] into the correct folders
-- Finish by deleting the temporary folder
-- ==================================================================
set AppleScript's text item delimiters to ":"
set td to (temp_dir as POSIX file) as string
set msg to "Searching " & td & " for files ..."
set filename to ""
my log_text(msg)
tell application "Finder" to set file_list to every file of folder td
if number of items in file_list > 0 then
repeat with i from 1 to number of items in file_list
set file_path to ((item i of file_list) as string)
set file_name to last text item of file_path
set fy to text 1 through 4 of file_name
set ch to text 30 through 30 of (file_name as string)
set msg to file_name & " ..."
my log_text(msg)
if ch is equal to "u" then
-- flux file
do shell script "cd " & quoted form of temp_dir & "; mv -f " & file_name & " ~/cassini_data/20day_FluxCalc/" & fy & "/flux_incr/"
set msg to "Moved " & file_name & " into ~/cassini_data/20day_FluxCalc/" & fy & "/flux_incr/"
my log_text(msg)
else if ch is equal to "M" then
-- mpqvt plot
do shell script "cd " & quoted form of temp_dir & "; curl --digest -u mimidata:chems4u -T " & file_name & " http://space.umd.edu/Projects/Cassini/chemsdata/chemsPHA/" & fy & "/"
set msg to "Uploaded http://space.umd.edu/Projects/Cassini/chemsdata/chemsPHA/" & fy & "/" & file_name
set emailContent to emailContent & "
" & msg
my log_text(msg)
do shell script "cd " & quoted form of temp_dir & "; mv -f " & file_name & " ~/cassini_data/MPQvs_t/" & fy & "/"
set msg to "Moved " & file_name & " into ~/cassini_data/MPQvs_t/" & fy & "/"
my log_text(msg)
else if ch is equal to "P" then
-- .xdr file
do shell script "cd " & quoted form of temp_dir & "; mv -f " & file_name & " ~/Documents/phaplot_xdr/"
set msg to "Moved " & file_name & " into ~/Documents/phaplot_xdr/"
my log_text(msg)
else
-- spectrogram
set s to text 29 through 29 of (file_name as string)
if s is not equal to "3" then
do shell script "cd " & quoted form of temp_dir & "; curl --digest -u mimidata:chems4u -T " & file_name & " http://space.umd.edu/Projects/Cassini/chemsdata/chemsscience/" & fy & "/"
set msg to "Uploaded http://space.umd.edu/Projects/Cassini/chemsdata/chemsscience/" & fy & "/" & file_name
set emailContent to emailContent & "
" & msg
my log_text(msg)
end if
do shell script "cd " & quoted form of temp_dir & "; mv -f " & file_name & " ~/cassini_data/20day_FluxCalc/" & fy & "/"
set msg to "Moved " & file_name & " into ~/cassini_data/20day_FluxCalc/" & fy & "/"
my log_text(msg)
end if
end repeat
-- Set the new "latest period"
set period_line to first paragraph of (read file "Macintosh HD:Users:stephen:local_chems_files:latest_period.txt" as {text})
set AppleScript's text item delimiters to " "
set period_data to every text item of period_line
log period_data
set period_number to ((item 1 of period_data) as string) & ((item 2 of period_data) as string)
log period_number
set new_number to (text 1 through 7 of file_name)
log new_number
# get the day of year for today
set nowdoy to (do shell script "date +%Y%j")
if (new_number > period_number) and (new_number < nowdoy) then
set period_number to new_number
set msg to "Set period number to " & period_number
set emailContent to emailContent & "
" & msg
my log_text(msg)
try
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:latest_period.txt") with write permission
on error number error_number
my log_text(error_number)
close access (file "Macintosh HD:Users:stephen:local_chems_files:latest_period.txt")
set write_file to open for access (file "Macintosh HD:Users:stephen:local_chems_files:latest_period.txt") with write permission
end try
set eof of write_file to 0
--write (text 1 through 4 of file_name) & " " & ((text 5 through 7 of file_name as integer) + periodoffset) to write_file
-- 7/21/15 sel remove periodoffset
write (text 1 through 4 of file_name) & " " & (text 5 through 7 of file_name) to write_file
close access write_file
end if
do shell script "cd /Users/stephen/local_chems_files; curl --digest -u mimidata:chems4u -T latest_period.txt http://space.umd.edu/Projects/Cassini/chemsdata/"
set msg to "Uploaded latest_period.txt"
my log_text(msg)
else
set msg to "Didn't find any files"
my log_text(msg)
end if
if new_files is equal to 1 then
do shell script "cd /Users/stephen/local_chems_files; curl --digest -u mimidata:chems4u -T newFiles.txt http://space.umd.edu/Projects/Cassini/chemsdata/"
set msg to "Uploaded newFiles.txt"
my log_text(msg)
end if
end if
-- Send daily email message
set dateToday to date string of (current date)
do shell script "cat << EOF | /usr/sbin/sendmail -t
From: CASSINI Mac
Subject: " & emailSubject & "
Date: " & dateToday & "
To: Douglas C. Hamilton
CC: Scott
" & emailContent & "
EOF"
-- Clean up after ourselves
set AppleScript's text item delimiters to ""
do shell script "rm -r " & quoted form of temp_dir
set msg to "Script complete"
my log_text(msg)