EOX GitLab Instance
Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
V
vs-deployment
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
This is an archived project. Repository and other project resources are read-only.
Show more breadcrumbs
Data Handling
vs-deployment
Commits
873834a9
Commit
873834a9
authored
2 years ago
by
Lubomir Dolezal
Browse files
Options
Downloads
Patches
Plain Diff
check that ingestor logs are not empty, autoformat
parent
5bb66838
No related branches found
Branches containing commit
No related tags found
Tags containing commit
1 merge request
!96
add full ingestor integration test
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
testing/test_ingestor.py
+28
-30
28 additions, 30 deletions
testing/test_ingestor.py
with
28 additions
and
30 deletions
testing/test_ingestor.py
+
28
−
30
View file @
873834a9
...
...
@@ -13,8 +13,11 @@ from swiftclient.service import SwiftService
def
set_gdal_swift_auth
():
# parsing command line output of swift auth
auth_keys
=
subprocess
.
check_output
([
"
swift
"
,
"
auth
"
]).
decode
(
sys
.
stdout
.
encoding
).
split
(
"
\n
"
)
auth_keys
=
(
subprocess
.
check_output
([
"
swift
"
,
"
auth
"
])
.
decode
(
sys
.
stdout
.
encoding
)
.
split
(
"
\n
"
)
)
storage_url
=
auth_keys
[
0
].
split
(
"
OS_STORAGE_URL=
"
)[
1
]
auth_token
=
auth_keys
[
1
].
split
(
"
OS_AUTH_TOKEN=
"
)[
1
]
# setting gdal config
...
...
@@ -24,8 +27,8 @@ def set_gdal_swift_auth():
def
list_dir_items
(
filename
):
output_list
=
[]
with
open
(
filename
,
newline
=
''
)
as
csvfile
:
spamreader
=
csv
.
reader
(
csvfile
,
delimiter
=
'
'
,
quotechar
=
'
|
'
)
with
open
(
filename
,
newline
=
""
)
as
csvfile
:
spamreader
=
csv
.
reader
(
csvfile
,
delimiter
=
"
"
,
quotechar
=
"
|
"
)
for
row
in
spamreader
:
for
item
in
row
:
output_list
.
append
(
item
)
...
...
@@ -34,22 +37,25 @@ def list_dir_items(filename):
def
test_ingestion_reporting
():
success_list
=
list_dir_items
(
'
success_list.csv
'
)
failure_list
=
list_dir_items
(
'
failure_list.csv
'
)
assert
'
tpzonlineitem.xml
'
in
success_list
assert
'
defected_report.xml
'
in
failure_list
success_list
=
list_dir_items
(
"
success_list.csv
"
)
failure_list
=
list_dir_items
(
"
failure_list.csv
"
)
assert
"
tpzonlineitem.xml
"
in
success_list
assert
"
defected_report.xml
"
in
failure_list
@pytest.fixture
def
list_tifs
(
product
=
'
KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar
'
):
def
list_tifs
(
product
=
"
KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar
"
,
):
preprocessed_list
=
[]
container
=
os
.
getenv
(
'
PREPROCESSOR_RESULTS_BUCKET
'
)
container
=
os
.
getenv
(
"
PREPROCESSOR_RESULTS_BUCKET
"
)
with
SwiftService
()
as
swift
:
# auth_options["prefix"] = product[0]
list_parts_gen
=
swift
.
list
(
container
=
container
,
options
=
{
"
prefix
"
:
product
[
0
]},
container
=
container
,
options
=
{
"
prefix
"
:
product
[
0
]},
)
for
page
in
list_parts_gen
:
if
page
[
"
success
"
]:
...
...
@@ -62,30 +68,30 @@ def list_tifs(product='KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_01933
def
test_preprocessor
(
list_tifs
):
# check if there are preprocessed results in the buckets
assert
len
(
list_tifs
)
>
0
container
=
os
.
getenv
(
'
PREPROCESSOR_RESULTS_BUCKET
'
)
container
=
os
.
getenv
(
"
PREPROCESSOR_RESULTS_BUCKET
"
)
for
item
in
list_tifs
:
set_gdal_swift_auth
()
image
=
gdal
.
Open
(
'
/vsiswift/%s/%s
'
%
(
container
,
item
),
gdal
.
GA_ReadOnly
)
image
=
gdal
.
Open
(
"
/vsiswift/%s/%s
"
%
(
container
,
item
),
gdal
.
GA_ReadOnly
)
srcband
=
image
.
GetRasterBand
(
1
)
assert
srcband
.
Checksum
()
is
not
None
assert
srcband
.
Checksum
()
>
0
def
test_error_logging
():
with
open
(
'
ingestor_log.txt
'
,
'
r
'
)
as
log_file
:
with
open
(
"
ingestor_log.txt
"
,
"
r
"
)
as
log_file
:
lines
=
log_file
.
read
().
splitlines
()
error_lines
=
list
(
filter
(
lambda
x
:
'
ERROR:
'
in
x
,
lines
))
xml_error_lines
=
list
(
filter
(
lambda
x
:
'
lxml.etree.XMLSyntaxError
'
in
x
,
lines
))
assert
len
(
lines
)
>
0
error_lines
=
list
(
filter
(
lambda
x
:
"
ERROR:
"
in
x
,
lines
))
xml_error_lines
=
list
(
filter
(
lambda
x
:
"
lxml.etree.XMLSyntaxError
"
in
x
,
lines
)
)
log_file
.
close
()
for
line
in
error_lines
:
# check the error notification
assert
'
Failed to parse XML.
'
in
line
match
=
re
.
search
(
'
[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}
'
,
line
)
date
=
datetime
.
datetime
.
strptime
(
match
.
group
(),
"
%Y-%m-%d %H:%M:%S
"
).
date
()
assert
"
Failed to parse XML.
"
in
line
match
=
re
.
search
(
"
[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}
"
,
line
)
date
=
datetime
.
datetime
.
strptime
(
match
.
group
(),
"
%Y-%m-%d %H:%M:%S
"
).
date
()
# check the associated error time
assert
type
(
date
)
==
datetime
.
date
...
...
@@ -93,11 +99,3 @@ def test_error_logging():
# Check the specific expected error reprot
error_msg
=
"
Start tag expected,
'
<
'
not found, line 1, column 1
"
assert
error_msg
in
line
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment