Merge pull request #491 from hormiai76/master

Remote energy + new statistics endpoint + some local fixes

Dockerfile

@@ -21,7 +21,7 @@ RUN mvn install:install-file -Dfile=dependencies/securityfilter.jar -DgroupId=se
     mvn install:install-file -Dfile=dependencies/jhdf5.jar -DgroupId=jhdf5 -DartifactId=jhdf5 -Dversion=1.0 -Dpackaging=jar && \
     mvn install:install-file -Dfile=dependencies/jhdf5obj.jar -DgroupId=jhdf5obj -DartifactId=jhdf5obj -Dversion=1.0 -Dpackaging=jar && \
     mvn install:install-file -Dfile=dependencies/jhdfobj.jar -DgroupId=jhdfobj -DartifactId=jhdfobj -Dversion=1.0 -Dpackaging=jar && \
-    mvn install:install-file -Dfile=dependencies/ispyb-userportal-gen-1.5.jar -DgroupId=ispyb -DartifactId=ispyb-userportal-gen -Dversion=1.5 -Dpackaging=jar && \
+    mvn install:install-file -Dfile=dependencies/ispyb-userportal-gen-1.7.jar -DgroupId=ispyb -DartifactId=ispyb-userportal-gen -Dversion=1.7 -Dpackaging=jar && \
     mvn install:install-file -Dfile=dependencies/Struts-Layout-1.2.jar -DgroupId=struts-layout -DartifactId=struts-layout -Dversion=1.2 -Dpackaging=jar && \
     mvn install -Dispyb.site=MAXIV -Dispyb.env=production && \
     cp ispyb-ear/target/ispyb.ear /var/ispyb.ear

ispyb-ejb/db/scripts/ahead/2020_05_08_EnergyScan_add_remoteEnergy.sql

@@ -0,0 +1,61 @@
+INSERT IGNORE INTO SchemaStatus (scriptName, schemaStatus) VALUES ('2020_05_08_EnergyScan_add_remoteEnergy.sql', 'ONGOING');
+
+ALTER TABLE `pydb`.`EnergyScan`
+ADD COLUMN `remoteEnergy` FLOAT NULL DEFAULT NULL AFTER `blSubSampleId`,
+ADD COLUMN `remoteFPrime` FLOAT NULL DEFAULT NULL AFTER `remoteEnergy`,
+ADD COLUMN `remoteFDoublePrime` FLOAT NULL DEFAULT NULL AFTER `remoteFPrime`;
+
+USE `pydb`;
+CREATE
+     OR REPLACE ALGORITHM = MERGE
+    DEFINER = `pxadmin`@`%`
+    SQL SECURITY DEFINER
+VIEW `v_energyScan` AS
+    SELECT
+        `EnergyScan`.`energyScanId` AS `energyScanId`,
+        `EnergyScan`.`sessionId` AS `sessionId`,
+        `EnergyScan`.`blSampleId` AS `blSampleId`,
+        `EnergyScan`.`fluorescenceDetector` AS `fluorescenceDetector`,
+        `EnergyScan`.`scanFileFullPath` AS `scanFileFullPath`,
+        `EnergyScan`.`choochFileFullPath` AS `choochFileFullPath`,
+        `EnergyScan`.`jpegChoochFileFullPath` AS `jpegChoochFileFullPath`,
+        `EnergyScan`.`element` AS `element`,
+        `EnergyScan`.`startEnergy` AS `startEnergy`,
+        `EnergyScan`.`endEnergy` AS `endEnergy`,
+        `EnergyScan`.`transmissionFactor` AS `transmissionFactor`,
+        `EnergyScan`.`exposureTime` AS `exposureTime`,
+        `EnergyScan`.`synchrotronCurrent` AS `synchrotronCurrent`,
+        `EnergyScan`.`temperature` AS `temperature`,
+        `EnergyScan`.`peakEnergy` AS `peakEnergy`,
+        `EnergyScan`.`peakFPrime` AS `peakFPrime`,
+        `EnergyScan`.`peakFDoublePrime` AS `peakFDoublePrime`,
+        `EnergyScan`.`inflectionEnergy` AS `inflectionEnergy`,
+        `EnergyScan`.`inflectionFPrime` AS `inflectionFPrime`,
+        `EnergyScan`.`inflectionFDoublePrime` AS `inflectionFDoublePrime`,
+        `EnergyScan`.`xrayDose` AS `xrayDose`,
+        `EnergyScan`.`startTime` AS `startTime`,
+        `EnergyScan`.`endTime` AS `endTime`,
+        `EnergyScan`.`edgeEnergy` AS `edgeEnergy`,
+        `EnergyScan`.`filename` AS `filename`,
+        `EnergyScan`.`beamSizeVertical` AS `beamSizeVertical`,
+        `EnergyScan`.`beamSizeHorizontal` AS `beamSizeHorizontal`,
+        `EnergyScan`.`crystalClass` AS `crystalClass`,
+        `EnergyScan`.`comments` AS `comments`,
+        `EnergyScan`.`flux` AS `flux`,
+        `EnergyScan`.`flux_end` AS `flux_end`,
+        `EnergyScan`.`remoteEnergy` AS `remoteEnergy`,
+        `EnergyScan`.`remoteFPrime` AS `remoteFPrime`,
+        `EnergyScan`.`remoteFDoublePrime` AS `remoteFDoublePrime`,
+        `BLSample`.`blSampleId` AS `BLSample_sampleId`,
+        `BLSample`.`name` AS `name`,
+        `BLSample`.`code` AS `code`,
+        `Protein`.`acronym` AS `acronym`,
+        `BLSession`.`proposalId` AS `BLSession_proposalId`
+    FROM
+        ((((`EnergyScan`
+        LEFT JOIN `BLSample` ON ((`BLSample`.`blSampleId` = `EnergyScan`.`blSampleId`)))
+        LEFT JOIN `Crystal` ON ((`Crystal`.`crystalId` = `BLSample`.`crystalId`)))
+        LEFT JOIN `Protein` ON ((`Protein`.`proteinId` = `Crystal`.`proteinId`)))
+        LEFT JOIN `BLSession` ON ((`BLSession`.`sessionId` = `EnergyScan`.`sessionId`)));
+
+UPDATE SchemaStatus SET schemaStatus = 'DONE' WHERE scriptName = '2020_05_08_EnergyScan_add_remoteEnergy.sql';

ispyb-ejb/db/scripts/ahead/2020_10_08_experiment_statistics_view.sql

@@ -0,0 +1,37 @@
+INSERT IGNORE INTO SchemaStatus (scriptName, schemaStatus) VALUES ('2020_10_08_experiment_statistics_view.sql', 'ONGOING');
+
+CREATE
+    ALGORITHM = UNDEFINED
+    DEFINER = `pxadmin`@`%`
+    SQL SECURITY DEFINER
+VIEW `v_mx_experiment_stats` AS
+    SELECT
+        `DC`.`startTime` AS `startTime`,
+        `DC`.`numberOfImages` AS `Images`,
+        `DC`.`transmission` AS `Transmission`,
+        `DC`.`resolution` AS `Res. (corner)`,
+        `DC`.`wavelength` AS `En. (Wave.)`,
+        `DC`.`omegaStart` AS `Omega start (total)`,
+        `DC`.`exposureTime` AS `Exposure Time`,
+        `DC`.`flux` AS `Flux`,
+        `DC`.`flux_end` AS `Flux End`,
+        `DC`.`detectorDistance` AS `Detector Distance`,
+        `DC`.`xBeam` AS `X Beam`,
+        `DC`.`yBeam` AS `Y Beam`,
+        `DC`.`kappaStart` AS `Kappa`,
+        `DC`.`phiStart` AS `Phi`,
+        `DC`.`axisStart` AS `Axis Start`,
+        `DC`.`axisEnd` AS `Axis End`,
+        `DC`.`axisRange` AS `Axis Range`,
+        `DC`.`beamSizeAtSampleX` AS `Beam Size X`,
+        `DC`.`beamSizeAtSampleY` AS `Beam Size Y`,
+        `BLS`.`beamLineName` AS `beamLineName`,
+        `DCG`.`comments` AS `comments`,
+        `P`.`proposalNumber` AS `proposalNumber`
+    FROM
+        (((`DataCollection` `DC`
+        JOIN `DataCollectionGroup` `DCG` ON ((`DCG`.`dataCollectionGroupId` = `DC`.`dataCollectionGroupId`)))
+        JOIN `BLSession` `BLS` ON ((`BLS`.`sessionId` = `DCG`.`sessionId`)))
+        JOIN `Proposal` `P` ON ((`P`.`proposalId` = `BLS`.`proposalId`)))
+
+UPDATE SchemaStatus SET schemaStatus = 'DONE' WHERE scriptName = '2020_10_08_experiment_statistics_view.sql';

ispyb-ejb/pom.xml

@@ -479,7 +479,7 @@
   			<ispyb.authentication.method>LDAP</ispyb.authentication.method>
   			<ispyb.authorisation.proposals.source>database</ispyb.authorisation.proposals.source>
 
-			<ldap.employee.resource>ldaps://srv-ad-2.maxiv.lu.se/</ldap.employee.resource>
+			<ldap.employee.resource>ldaps://adauth.maxiv.lu.se/</ldap.employee.resource>
 			<ldap.base>dc=maxlab,dc=lu,dc=se</ldap.base>
 			<ldap.people>dc=maxlab,dc=lu,dc=se</ldap.people>
 			<ldap.attribute>samaccountname</ldap.attribute>

ispyb-ejb/src/main/java/ispyb/common/util/export/ExiPdfRtfExporter.java

@@ -810,7 +810,10 @@ private void setEnergyScanMapData(Document document, Map<String, Object> energyS
 				+ 	"Pk f'': \n" 
 				+ 	"Inflection Energy: \n" 
 				+ 	"Ip f': \n" 
-				+ 	"Ip f'': \n" ;
+				+ 	"Ip f'': \n"
+				+ 	"Remote Energy: \n"
+				+ 	"Remote f': \n"
+				+ 	"Remote f'': \n" ;
 
 		table.addCell(new Paragraph(parag, FONT_DOC));
 
@@ -819,9 +822,12 @@ private void setEnergyScanMapData(Document document, Map<String, Object> energyS
 		parag = getCellParam(energyScanMapItem, "peakEnergy", df3) + "keV \n"
 			+ getCellParam(energyScanMapItem, "peakFPrime", df2) + " e- \n" 
 		+ 	getCellParam(energyScanMapItem, "peakFDoublePrime", df2) + " e- \n" 
-		+  "("+ getCellParam(energyScanMapItem, "inflectionEnergy", df2) + "keV \n" 
+		+   getCellParam(energyScanMapItem, "inflectionEnergy", df2) + "keV \n"
 		+ 	getCellParam(energyScanMapItem, "inflectionFPrime", df2)  + " e- \n" 
-		+ 	getCellParam(energyScanMapItem, "inflectionFDoublePrime", df2)  + " e- \n" ;
+		+ 	getCellParam(energyScanMapItem, "inflectionFDoublePrime", df2)  + " e- \n"
+		+   getCellParam(energyScanMapItem, "remoteEnergy", df2) + "keV \n"
+		+ 	getCellParam(energyScanMapItem, "remoteFPrime", df2)  + " e- \n"
+		+ 	getCellParam(energyScanMapItem, "remoteFDoublePrime", df2)  + " e- \n" ;;
 
 		table.addCell(new Paragraph(parag, FONT_DOC_BOLD));
 
@@ -1173,7 +1179,7 @@ private void setEnergyScanMapData2(Document document, Map<String, Object> energy
 		String parag = getCellParam(energyScanMapItem, "scanFileFullPath", null) + "\n" ;
 		document.add(new Paragraph(parag, FONT_DOC_SMALL));
 
-		Table table = new Table(9);
+		Table table = new Table(11);
 		table.getDefaultCell().setHorizontalAlignment(Element.ALIGN_LEFT);
 		table.getDefaultCell().setBorderWidth(0);
 		table.setBorder(0);
@@ -1250,7 +1256,18 @@ private void setEnergyScanMapData2(Document document, Map<String, Object> energy
 		parag = getCellParam(energyScanMapItem, "inflectionEnergy", df2) + "keV \n" 
 		+ 	getCellParam(energyScanMapItem, "inflectionFPrime", df2)  + " e- \n" 
 		+ 	getCellParam(energyScanMapItem, "inflectionFDoublePrime", df2)  + " e- \n" ;
-
+
+		// Cell 10
+		parag = "Remote Energy: \n"
+				+ 	"Remote f': \n"
+				+ 	"Remote f'': \n" ;
+
+		table.addCell(new Paragraph(parag, FONT_DOC_SMALL));
+
+		// Cell 11
+		parag = getCellParam(energyScanMapItem, "remoteEnergy", df2) + "keV \n"
+				+ 	getCellParam(energyScanMapItem, "remoteFPrime", df2)  + " e- \n"
+				+ 	getCellParam(energyScanMapItem, "remoteFDoublePrime", df2)  + " e- \n" ;
 		table.addCell(new Paragraph(parag, FONT_DOC_SMALL_BOLD));
 
 		document.add(table);

ispyb-ejb/src/main/java/ispyb/server/biosaxs/services/stats/Stats3Service.java

@@ -45,6 +45,8 @@ public interface Stats3Service {
 
 	List<Map<String, Object>> getDatacollectionStatsByDate(String datacollectionStatisticsType, Date startDate, Date endDate, String[] datacollectionTestProposals, String beamline);
 
+	List<Map<String, Object>> getExperimentStatsByDate(Date startDate, Date endDate, String[] datacollectionTestProposals);
 
+	List<Map<String, Object>> getExperimentStatsByDate(Date startDate, Date endDate, String[] datacollectionTestProposals, String beamline);
 
 }

ispyb-ejb/src/main/java/ispyb/server/biosaxs/services/stats/Stats3ServiceBean.java

@@ -62,6 +62,8 @@ public class Stats3ServiceBean extends WsServiceBean implements Stats3Service,
 												+"and DCG.comments not in (' Data collection failed!\n') "
 												+"and P.proposalNumber not in (:TESTPROPOSALS) ";
 
+	private String EXPERIMENTSTATS_QUERY ="select * from v_mx_experiment_stats where startTime >= :START and startTime <= :END and comments not in (' Data collection failed!\\n') and proposalNumber not in (:TESTPROPOSALS) ";
+
 	@PersistenceContext(unitName = "ispyb_db")
 	private EntityManager entityManager;
 
@@ -153,4 +155,29 @@ public List<Map<String, Object>> getDatacollectionStatsByDate(
 		query.setParameter("BEAMLINENAME",beamline);
 		return executeSQLQuery(query);
 	}
+
+	@Override
+	public List<Map<String, Object>> getExperimentStatsByDate(
+			Date startDate, Date endDate, String[] datacollectionTestProposals) {
+		Session session = (Session) this.entityManager.getDelegate();
+		SQLQuery query = session.createSQLQuery(EXPERIMENTSTATS_QUERY);
+		SimpleDateFormat dt1 = new SimpleDateFormat("yyyy-MM-dd");
+		query.setParameter("START", dt1.format(startDate));
+		query.setParameter("END", dt1.format(endDate));
+		query.setParameterList("TESTPROPOSALS", datacollectionTestProposals);
+		return executeSQLQuery(query);
+	}
+
+	@Override
+	public List<Map<String, Object>> getExperimentStatsByDate(
+			Date startDate, Date endDate, String[] datacollectionTestProposals, String beamline) {
+		Session session = (Session) this.entityManager.getDelegate();
+		SQLQuery query = session.createSQLQuery(EXPERIMENTSTATS_QUERY + "  and beamLineName = :BEAMLINENAME ");
+		SimpleDateFormat dt1 = new SimpleDateFormat("yyyy-MM-dd");
+		query.setParameter("START", dt1.format(startDate));
+		query.setParameter("END", dt1.format(endDate));
+		query.setParameterList("TESTPROPOSALS", datacollectionTestProposals);
+		query.setParameter("BEAMLINENAME",beamline);
+		return executeSQLQuery(query);
+	}
 }

ispyb-ejb/src/main/java/ispyb/server/mx/services/utils/reader/AutoProcProgramaAttachmentFileReader.java

@@ -35,7 +35,7 @@ public static HashMap<String, Object> readAttachment(AutoProcProgramAttachment3V
 			String fileName = attachment.getFileName();
 			xscaleFile = (fileName != null && fileName.toLowerCase().endsWith("xscale.lp"));
 			truncateLog = (fileName != null && fileName.toLowerCase().endsWith(".log") && fileName.toLowerCase().contains("truncate"));
-			noanomAimlessLog = (fileName != null && fileName.toLowerCase().endsWith(".log") && fileName.toLowerCase().contains("noanom_aimless"));
+			noanomAimlessLog = (fileName != null && fileName.toLowerCase().endsWith(".log") && fileName.toLowerCase().contains("aimless"));
 
 //			System.out.println(xscaleFile);
 //			System.out.println(truncateLog);

ispyb-ejb/src/main/java/ispyb/server/mx/services/ws/rest/datacollectiongroup/DataCollectionGroupRestWsService.java

@@ -34,6 +34,8 @@ public interface DataCollectionGroupRestWsService {
 	public List<Map<String, Object>> getViewDataCollectionBySessionIdHavingImages(int proposalId, int sessionId);
 
 	public List<Map<String, Object>> getViewDataCollectionByProteinAcronym(int proposalId, String proteinAcronym);
+
+	public List<Map<String, Object>> getViewDataCollectionBySampleId(int proposalId, int sampleId);
 
 	public List<Map<String, Object>> getViewDataCollectionBySampleName(int proposalId, String name);
 

ispyb-ejb/src/main/java/ispyb/server/mx/services/ws/rest/datacollectiongroup/DataCollectionGroupRestWsServiceBean.java

@@ -80,6 +80,18 @@ public List<Map<String, Object>> getViewDataCollectionByProteinAcronym(int propo
 		return executeSQLQuery(query);
 	}
 
+	@Override
+	public List<Map<String, Object>> getViewDataCollectionBySampleId(int proposalId, int sampleId) {
+		String mySQLQuery = getViewTableQuery() + " where BLSession_proposalId = :proposalId and DataCollectionGroup_blSampleId = :sampleId";
+		mySQLQuery = mySQLQuery + " group by v_datacollection_summary.DataCollectionGroup_dataCollectionGroupId, v_datacollection_summary.DataCollectionGroup_dataCollectionGroupId";
+		Session session = (Session) this.entityManager.getDelegate();
+		SQLQuery query = session.createSQLQuery(mySQLQuery);
+		query.setParameter("proposalId", proposalId);
+		query.setParameter("sampleId", sampleId);
+		query.setResultTransformer(AliasToEntityMapResultTransformer.INSTANCE);
+		return executeSQLQuery(query);
+	}
+
 	@Override
 	public List<Map<String, Object>> getViewDataCollectionBySampleName(int proposalId, String name) {
 		String mySQLQuery = getViewTableQuery() + " where BLSession_proposalId = :proposalId and BLSample_name = :name";

ispyb-ejb/src/main/java/ispyb/server/mx/vos/collections/EnergyScan3VO.java

@@ -149,6 +149,15 @@ public class EnergyScan3VO extends ISPyBValueObject implements Cloneable {
 
 	@Column(name = "workingDirectory")
 	protected String workingDirectory;
+
+	@Column(name = "remoteEnergy")
+	protected Double remoteEnergy;
+
+	@Column(name = "remoteFPrime")
+	protected Double remoteFPrime;
+
+	@Column(name = "remoteFDoublePrime")
+	protected Double remoteFDoublePrime;
 
 	@ManyToOne
 	@JoinColumn(name = "blSubSampleId")
@@ -169,7 +178,8 @@ public EnergyScan3VO(Integer energyScanId, Session3VO sessionVO, BLSample3VO sam
 			Double inflectionFDoublePrime, Double xrayDose, Date startTime,
 			Date endTime, String edgeEnergy, String filename,
 			Float beamSizeVertical, Float beamSizeHorizontal,
-			String crystalClass, String comments, Double flux, Double flux_end) {
+			String crystalClass, String comments, Double flux, Double flux_end,
+			Double remoteEnergy, Double remoteFPrime, Double remoteFDoublePrime) {
 		super();
 		this.energyScanId = energyScanId;
 		this.sessionVO = sessionVO;
@@ -202,6 +212,9 @@ public EnergyScan3VO(Integer energyScanId, Session3VO sessionVO, BLSample3VO sam
 		this.comments = comments;
 		this.flux = flux;
 		this.flux_end = flux_end;
+		this.remoteEnergy = remoteEnergy;
+		this.remoteFPrime = remoteFPrime;
+		this.remoteFDoublePrime = remoteFDoublePrime;
 	}
 
 
@@ -238,6 +251,9 @@ public EnergyScan3VO(EnergyScan3VO vo) {
 		this.comments = vo.getComments();
 		this.flux = vo.getFlux();
 		this.flux_end = vo.getFlux_end();
+		this.remoteEnergy = vo.getRemoteEnergy();
+		this.remoteFPrime = vo.getRemoteFPrime();
+		this.remoteFDoublePrime = vo.getRemoteFDoublePrime();
 	}
 
 
@@ -274,6 +290,9 @@ public void fillVOFromLight(EnergyScanWS3VO vo) {
 		this.comments = vo.getComments();
 		this.flux = vo.getFlux();
 		this.flux_end = vo.getFlux_end();
+		this.remoteEnergy = vo.getRemoteEnergy();
+		this.remoteFPrime = vo.getRemoteFPrime();
+		this.remoteFDoublePrime = vo.getRemoteFDoublePrime();
 	}
 
 
@@ -555,6 +574,24 @@ public void setFlux_end(Double flux_end) {
 		this.flux_end = flux_end;
 	}
 
+	public Double getRemoteEnergy() { return remoteEnergy; }
+	public void setRemoteEnergy(Double remoteEnergy) {
+		this.remoteEnergy = remoteEnergy;
+	}
+
+	public Double getRemoteFPrime() {
+		return remoteFPrime;
+	}
+	public void setRemoteFPrime(Double remoteFPrime) {
+		this.remoteFPrime = remoteFPrime;
+	}
+
+	public Double getRemoteFDoublePrime() {
+		return remoteFDoublePrime;
+	}
+	public void setRemoteFDoublePrime(Double remoteFDoublePrime) {
+		this.remoteFDoublePrime = remoteFDoublePrime;
+	}
 
 
 	/**
@@ -637,7 +674,10 @@ public String toWSString(){
 		"crystalClass="+this.crystalClass+", "+
 		"comments="+this.comments+", "+
 		"flux="+this.flux+", "+
-		"flux_end="+this.flux_end;
+		"flux_end="+this.flux_end+", "+
+		"remoteEnergy="+this.remoteEnergy+", "+
+		"remoteFPrime="+this.remoteFPrime+", "+
+		"remoteFDoublePrime="+this.remoteFDoublePrime;
 
 		return s;
 	}

ispyb-ejb/src/main/java/ispyb/server/security/LdapLoginModule.java

@@ -283,6 +283,7 @@ private void createLdapInitContext(String username, Object credential) throws Na
 			env.put(Context.SECURITY_AUTHENTICATION, "simple");
 			env.put("jboss.security.security_domain", "ispyb");
 			env.put("allowEmptyPasswords", "false");
+			//LOG.info("Env:" + env);
 		}
 
 
@@ -293,8 +294,8 @@ private void createLdapInitContext(String username, Object credential) throws Na
 		try {
 			ctx = new InitialLdapContext(env, null);
 		}catch (Exception ex){
-			if (username.equals("ispyb")) {
-				LOG.debug("Env:" + env);
+			if (Constants.SITE_IS_MAXIV() && username.equals("ispyb")) {
+				LOG.info("Env:" + env);
 			}
 			throw ex;
 		}
@@ -366,12 +367,12 @@ private void createLdapInitContext(String username, Object credential) throws Na
 						Attribute roles = attrs.get(groupAttrName);
 
 						for (int r = 0; r < roles.size(); r++) {
-
 							Object value = roles.get(r);
 							String roleName = null;
 							roleName = value.toString();
 							// fill roles array
 							if (roleName != null) {
+								LOG.info("Role found for " +username +":" +roleName);
 								if (roleName.equals("ispyb-manager") || roleName.equals("ispyb-biomax-contacts") ||
 										roleName.equals("Information Management") || roleName.equals("biomax")) {
 									userRoles.addMember(new SimplePrincipal(Constants.ROLE_MANAGER));