Merge pull request #106 from PanDAWMS/v0.0.29

V0.0.29

Author: EdwardKaravakis

schema/oracle/ATLAS_PANDA.sql

@@ -34,8 +34,8 @@
 --  IMPORTANT: Please always update to up2date version
 --------------------------------------------------------
 
-  INSERT INTO "ATLAS_PANDA"."PANDADB_VERSION" VALUES ('SERVER', 0, 0, 28);
-  INSERT INTO "ATLAS_PANDA"."PANDADB_VERSION" VALUES ('JEDI', 0, 0, 28);
+  INSERT INTO "ATLAS_PANDA"."PANDADB_VERSION" VALUES ('SERVER', 0, 0, 29);
+  INSERT INTO "ATLAS_PANDA"."PANDADB_VERSION" VALUES ('JEDI', 0, 0, 29);
 
  --------------------------------------------------------
 --  DDL for Sequence FILESTABLE4_ROW_ID_SEQ
@@ -203,7 +203,10 @@ CREATE TABLE "ATLAS_PANDA"."JEDI_DATASET_LOCALITY"
 	"BLACKLISTED_READ" CHAR(1 BYTE),
 	"BLACKLISTED_WRITE" CHAR(1 BYTE),
 	"SPACE_EXPIRED" NUMBER(11,0), 
-	"SPACE_TIMESTAMP" TIMESTAMP (0)
+	"SPACE_TIMESTAMP" TIMESTAMP (0),
+     "DETAILED_STATUS"   CLOB
+       CONSTRAINT detailed_status_is_json
+       CHECK ("DETAILED_STATUS" IS JSON) ENABLE
    ) ;
 
    COMMENT ON COLUMN "ATLAS_PANDA"."DDM_ENDPOINT"."DDM_ENDPOINT_NAME" IS 'DDM endpoint name';
@@ -219,6 +222,7 @@ CREATE TABLE "ATLAS_PANDA"."JEDI_DATASET_LOCALITY"
    COMMENT ON COLUMN "ATLAS_PANDA"."DDM_ENDPOINT"."BLACKLISTED_WRITE" IS 'Defines whether a DDM endpoint is blacklisted for write (Y/N)';
    COMMENT ON COLUMN "ATLAS_PANDA"."DDM_ENDPOINT"."SPACE_EXPIRED" IS 'Expired, used space in GB';
    COMMENT ON COLUMN "ATLAS_PANDA"."DDM_ENDPOINT"."SPACE_TIMESTAMP" IS 'Timestamp reported by Rucio for the SRM space values';
+   COMMENT ON COLUMN "ATLAS_PANDA"."DDM_ENDPOINT"."DETAILED_STATUS" IS 'Endpoint-specific detailed status (JSON)';
    COMMENT ON TABLE "ATLAS_PANDA"."DDM_ENDPOINT"  IS 'DDM/Rucio storage endpoint.Table needed for the "Configurator" agent. The table consolidate information from other sources (AGIS and Rucio ) and caches it so that Panda can easily retrieve it and use it for brokerage.';
 
 
@@ -691,9 +695,9 @@ CREATE TABLE "ATLAS_PANDA"."GLOBAL_SHARES_AUDIT"
 	"MAXATTEMPT" NUMBER(3,0), 
 	"NEVENTS" NUMBER(10,0), 
 	"KEEPTRACK" NUMBER(1,0), 
-	"STARTEVENT" NUMBER(10,0), 
-	"ENDEVENT" NUMBER(10,0), 
-	"FIRSTEVENT" NUMBER(10,0), 
+	"STARTEVENT" NUMBER(11,0),
+	"ENDEVENT" NUMBER(11,0),
+	"FIRSTEVENT" NUMBER(11,0),
 	"BOUNDARYID" NUMBER(11,0), 
 	"PANDAID" NUMBER(11,0), 
 	"FAILEDATTEMPT" NUMBER(3,0), 
@@ -3066,6 +3070,7 @@ INTERVAL (NUMTOYMINTERVAL(1, 'MONTH')) (
 );
 
 CREATE INDEX mn_metrics_idx ON "ATLAS_PANDA"."WORKER_NODE_METRICS"("SITE", "HOST_NAME", "TIMESTAMP");
+CREATE INDEX "WN_METRICS_TIMESTAMP_IDX" ON "ATLAS_PANDA"."WORKER_NODE_METRICS" ("TIMESTAMP");
 
 -- Table Comment
 COMMENT ON TABLE "ATLAS_PANDA"."WORKER_NODE_METRICS" IS 'Metrics related to a worker node';

schema/postgres/sqls/patches/0.0.29.patch.sql

@@ -0,0 +1,20 @@
+-- patch to be used to upgrade from version 0.0.28
+CREATE INDEX wn_metrics_timestamp_idx ON doma_panda.worker_node_metrics ("timestamp");
+
+ALTER TABLE doma_panda.ddm_endpoint
+  ADD COLUMN IF NOT EXISTS detailed_status JSONB;
+
+COMMENT ON COLUMN doma_panda.ddm_endpoint.detailed_status
+  IS 'Endpoint-specific detailed status (JSON)';
+
+
+-- Update schema version
+UPDATE doma_panda.pandadb_version
+SET major = 0, minor = 0, patch = 29
+WHERE component = 'JEDI';
+
+UPDATE doma_panda.pandadb_version
+SET major = 0, minor = 0, patch = 29
+WHERE component = 'SERVER';
+
+commit;

schema/postgres/sqls/pg_PANDA_SCHEDULER_JOBS.sql

@@ -2491,6 +2491,7 @@ BEGIN
         WHERE j."endtime" > NOW() - INTERVAL '1 day'
           AND j."jobstatus" IN ('finished', 'failed')
           AND j."modificationhost" NOT LIKE 'aipanda%'
+          AND j."modificationhost" NOT LIKE 'grid-job-%'
         GROUP BY sc_slimmed."atlas_site", "worker_node"
     ),
     harvester_stats AS (
@@ -2510,6 +2511,7 @@ BEGIN
         JOIN sc_slimmed ON h."computingsite" = sc_slimmed."panda_queue"
         WHERE h."endtime" > NOW() - INTERVAL '1 day'
           AND h."status" IN ('finished', 'failed', 'cancelled')
+          AND h."nodeid" NOT LIKE 'grid-job-%'
         GROUP BY sc_slimmed."atlas_site", "worker_node"
     )
     SELECT "atlas_site", "worker_node", "key", "stats" FROM pilot_stats

schema/postgres/sqls/pg_PANDA_TABLE.sql

@@ -144,7 +144,8 @@ CREATE TABLE ddm_endpoint (
 	space_expired bigint,
 	space_timestamp timestamp,
 	blacklisted_read char(1),
-	blacklisted_write char(1)
+	blacklisted_write char(1),
+    detailed_status JSONB
 ) ;
 COMMENT ON TABLE ddm_endpoint IS E'DDM/Rucio storage endpoint.Table needed for the "Configurator" agent. The table consolidate information from other sources (AGIS and Rucio ) and caches it so that Panda can easily retrieve it and use it for brokerage.';
 COMMENT ON COLUMN ddm_endpoint.blacklisted IS E'Defines whether a DDM endpoint is blacklisted or not (Y/N)';
@@ -160,6 +161,7 @@ COMMENT ON COLUMN ddm_endpoint.space_timestamp IS E'Timestamp reported by Rucio
 COMMENT ON COLUMN ddm_endpoint.space_total IS E'Total space of a DDM endpoint as reported by Rucio. Value in GB';
 COMMENT ON COLUMN ddm_endpoint.space_used IS E'Used space of a DDM endpoint as reported by Rucio. Value in GB';
 COMMENT ON COLUMN ddm_endpoint.type IS E'Type of spacetoken, e.g. DATADISK, LOCALGROUPDISK...';
+COMMENT ON COLUMN ddm_endpoint.detailed_status IS 'Endpoint-specific detailed status (JSON)';
 ALTER  TABLE ddm_endpoint OWNER TO panda;
 CREATE INDEX ddm_endpoint_site_name_idx ON ddm_endpoint (site_name);
 ALTER TABLE ddm_endpoint ADD PRIMARY KEY (ddm_endpoint_name);
@@ -2995,6 +2997,8 @@ COMMENT ON COLUMN worker_node_metrics."key" IS 'Key of the metrics entry.';
 COMMENT ON COLUMN worker_node_metrics."statistics" IS 'Metrics in json format.';
 
 CREATE INDEX wn_metrics_idx ON worker_node_metrics ("site", "host_name", "timestamp");
+CREATE INDEX wn_metrics_timestamp_idx ON worker_node_metrics ("timestamp");
+
 ALTER TABLE worker_node_metrics OWNER TO panda;
 
 CREATE TABLE error_descriptions (

schema/postgres/version

@@ -1 +1 @@
-0.0.28
+0.0.29