diff --git a/task-1/output/clean_sales.csv b/task-1/output/clean_sales.csv new file mode 100644 index 0000000..be91924 --- /dev/null +++ b/task-1/output/clean_sales.csv @@ -0,0 +1,13 @@ +transaction_id,product_name,category,price,quantity,customer_email,date,revenue,vat +1,Laptop Pro,Electronics,999.99,2,alice@example.com,2024-03-15,1999.98,420.0 +2,Wireless Mouse,Electronics,29.99,5,bob@company.com,2024-03-15,149.95,31.49 +3,Usb Cable,Electronics,4.99,10,,2024-03-16,49.9,10.48 +4,Office Chair,Furniture,349.5,1,charlie@work.org,2024-03-16,349.5,73.39 +5,Standing Desk,Furniture,599.0,1,charlie@work.org,not_a_date,599.0,125.79 +9,Webcam Hd,Electronics,54.99,1,,2024-03-18,54.99,11.55 +10,Desk Lamp,Furniture,34.99,4,grace@university.edu,2024-03-19,139.96,29.39 +11,Noise Cancelling Headphones,Electronics,199.99,1,alice@example.com,2024-03-19,199.99,42.0 +12,Cable Management Kit,Furniture,15.99,6,henry@business.com,2024-03-20,95.94,20.15 +13,Ergonomic Mouse Pad,Furniture,24.99,3,ivan@email.com,2024-03-20,74.97,15.74 +14,Laptop Stand,Furniture,45.99,2,jenny@work.org,2024-03-21,91.98,19.32 +15,Bluetooth Speaker,Unknown,39.99,1,karl@startup.io,2024-03-21,39.99,8.4 diff --git a/task-1/src/__pycache__/__init__.cpython-311.pyc b/task-1/src/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..79755f2 Binary files /dev/null and b/task-1/src/__pycache__/__init__.cpython-311.pyc differ diff --git a/task-1/src/__pycache__/config.cpython-311.pyc b/task-1/src/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000..f11103c Binary files /dev/null and b/task-1/src/__pycache__/config.cpython-311.pyc differ diff --git a/task-1/src/__pycache__/models.cpython-311.pyc b/task-1/src/__pycache__/models.cpython-311.pyc new file mode 100644 index 0000000..72af137 Binary files /dev/null and b/task-1/src/__pycache__/models.cpython-311.pyc differ diff --git a/task-1/src/__pycache__/pipeline.cpython-311.pyc b/task-1/src/__pycache__/pipeline.cpython-311.pyc new file mode 100644 index 0000000..ee51a5a Binary files /dev/null and b/task-1/src/__pycache__/pipeline.cpython-311.pyc differ diff --git a/task-1/src/__pycache__/transforms.cpython-311.pyc b/task-1/src/__pycache__/transforms.cpython-311.pyc new file mode 100644 index 0000000..e60248b Binary files /dev/null and b/task-1/src/__pycache__/transforms.cpython-311.pyc differ diff --git a/task-1/src/config.py b/task-1/src/config.py index e59f7c1..ce4f823 100644 --- a/task-1/src/config.py +++ b/task-1/src/config.py @@ -17,19 +17,28 @@ # (Step 1 from the docstring above; already wired up so the rest of the # module can rely on os.environ being populated.) load_dotenv() +input_path = (os.environ["INPUT_PATH"]) +output_path =(os.environ["OUTPUT_PATH"]) def _required(name: str) -> str: """Read an env var; fail loudly if missing.""" # TODO 2: Read os.environ[name]; if not set, raise ValueError with a # message that names the missing variable AND points at .env.example. - raise NotImplementedError("Implement _required: see TODO 2 in config.py") + value = os.environ.get(name) + if not value: + raise ValueError(f"Missing required environment variable: {name}") + return value +if not input_path: + raise ValueError("Missing required environment variable: INPUT_PATH") +if not output_path: + raise ValueError("Missing required environment variable: OUTPUT_PATH") # TODO 3: Replace the placeholder lines below by calling _required(...) for # each variable. INPUT_PATH and OUTPUT_PATH must be importable from this # module by the rest of the pipeline as a relative import # (`from .config import INPUT_PATH, ...`), since the pipeline runs as # `python -m src.pipeline`. -INPUT_PATH: str = "" # TODO: _required("INPUT_PATH") -OUTPUT_PATH: str = "" # TODO: _required("OUTPUT_PATH") +INPUT_PATH: str = _required("INPUT_PATH") # TODO: _required("INPUT_PATH") +OUTPUT_PATH: str = _required("OUTPUT_PATH") # TODO: _required("OUTPUT_PATH") diff --git a/task-1/src/models.py b/task-1/src/models.py index 865b2b1..2a173ad 100644 --- a/task-1/src/models.py +++ b/task-1/src/models.py @@ -30,4 +30,19 @@ # Replace this stub with your dataclass: @dataclass class Transaction: - transaction_id: int # TODO: replace this stub with the full field list above + transaction_id: int + product_name: str # TODO: replace this stub with the full field list above + category: str + price: float + quantity: int + customer_email: str + date: str + revenue: float = 0.0 + vat: float = 0.0 + + def __post_init__(self): + if (self.price <= 0): + raise ValueError(f"{self.price} must be greater than zero ") + + if (not self.product_name.strip()): + raise ValueError(f"{self.product_name} cannot be empty") \ No newline at end of file diff --git a/task-1/src/pipeline.py b/task-1/src/pipeline.py index adce136..868075b 100644 --- a/task-1/src/pipeline.py +++ b/task-1/src/pipeline.py @@ -32,13 +32,20 @@ def read_csv(path: str) -> list[dict]: """Read a CSV file into a list of dicts. I/O only — no business rules.""" # TODO: implement using csv.DictReader. + with open(path, mode='r', encoding='utf-8') as file: + reader = csv.DictReader(file) + return list(reader) raise NotImplementedError def write_csv(rows: list[dict], path: str) -> None: """Write a list of dicts to CSV. I/O only — no business rules.""" # TODO: implement using csv.DictWriter. - raise NotImplementedError + with open(path, mode='w', encoding='utf-8', newline='') as file: + if rows: + writer = csv.DictWriter(file, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(rows) def run() -> None: @@ -50,10 +57,16 @@ def run() -> None: # Materialise as Transaction instances so the dataclass __post_init__ # acts as a final guard before serialisation. - # TODO: cast price / quantity / revenue / vat to the right types here - # if your transforms left them as strings, then iterate over `data` - # to build Transaction(**row) for each cleaned row. - transactions = [Transaction(**row) for row in data] + # The transforms layer returns cleaned dicts; iterate over them to build + # Transaction instances, which validates all fields in __post_init__. + transactions = [] + for row in data: + # Cast string fields to correct types before creating Transaction + row['price'] = float(row['price']) + row['quantity'] = int(row['quantity']) + row['revenue'] = float(row['revenue']) + row['vat'] = float(row['vat']) + transactions.append(Transaction(**row)) # Output dir must exist. Use pathlib for cross-platform safety. Path(OUTPUT_PATH).parent.mkdir(parents=True, exist_ok=True) diff --git a/task-1/src/transforms.py b/task-1/src/transforms.py index e6bdd76..264642e 100644 --- a/task-1/src/transforms.py +++ b/task-1/src/transforms.py @@ -19,8 +19,12 @@ def remove_invalid(rows: list[dict]) -> list[dict]: Empty here means missing, "", or whitespace-only. """ # TODO: implement. Return a new list, do not mutate `rows`. - raise NotImplementedError - + + return [ + row for row in rows + if row.get("product_name", "").strip() + and float(row.get("price", 0)) >= 0 + ] def clean_fields(rows: list[dict]) -> list[dict]: """Clean string fields: @@ -32,7 +36,15 @@ def clean_fields(rows: list[dict]) -> list[dict]: Return a new list. Do not mutate the input rows. """ # TODO: implement. - raise NotImplementedError + return [ + { + **row, # ✅ spread pattern - copies all existing fields + "product_name": row.get("product_name", "").strip().title(), + "customer_email": row.get("customer_email", "").strip().lower(), + "category": row.get("category", "").strip() or "Unknown", + } + for row in rows + ] def calculate_revenue(rows: list[dict], vat_rate: float = 0.21) -> list[dict]: @@ -41,10 +53,20 @@ def calculate_revenue(rows: list[dict], vat_rate: float = 0.21) -> list[dict]: Round both to 2 decimal places. Coerce price/quantity from string if needed. """ # TODO: implement. - raise NotImplementedError - + return [ + { + **row, + "revenue": round(float(row.get("price", 0)) * int(row.get("quantity", 0)), 2), + "vat": round(float(row.get("price", 0)) * int(row.get("quantity", 0)) * vat_rate, 2), + } + for row in rows + ] + def filter_zero_quantity(rows: list[dict]) -> list[dict]: """Remove rows where quantity is 0.""" # TODO: implement. - raise NotImplementedError + return [ + row for row in rows + if int(row.get("quantity", 0)) != 0 + ] \ No newline at end of file diff --git a/task-1/tests/__pycache__/__init__.cpython-311.pyc b/task-1/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..9ec5a74 Binary files /dev/null and b/task-1/tests/__pycache__/__init__.cpython-311.pyc differ diff --git a/task-1/tests/__pycache__/test_transforms.cpython-311-pytest-9.0.3.pyc b/task-1/tests/__pycache__/test_transforms.cpython-311-pytest-9.0.3.pyc new file mode 100644 index 0000000..b688697 Binary files /dev/null and b/task-1/tests/__pycache__/test_transforms.cpython-311-pytest-9.0.3.pyc differ diff --git a/task-1/tests/test_transforms.py b/task-1/tests/test_transforms.py index 8d6bd80..85e851d 100644 --- a/task-1/tests/test_transforms.py +++ b/task-1/tests/test_transforms.py @@ -19,22 +19,42 @@ def test_remove_invalid_drops_empty_names(): # TODO: feed in 3 rows (one valid, one empty product_name, one # whitespace-only product_name); assert only the valid one survives. - raise NotImplementedError + data = [ + {"product_name": "Laptop", "price": 999.99}, + {"product_name": "", "price": 50.0}, + {"product_name": " ", "price": 25.0}, + ] + result = remove_invalid(data) + assert len(result) == 1 + assert result[0]["product_name"] == "Laptop" + def test_clean_fields_normalizes_names(): # TODO: feed a row with messy product_name and uppercase email; assert # the output has stripped + title-cased name and lowercase email. - raise NotImplementedError + data = [{"product_name": " laptop ", "customer_email": "JOHN@EXAMPLE.COM"}] + result = clean_fields(data) + assert result[0]["product_name"] == "Laptop" + assert result[0]["customer_email"] == "john@example.com" + def test_calculate_revenue_adds_fields(): # TODO: feed a row with price=100, quantity=3; assert output has # revenue=300.0 and vat=63.0 (default VAT rate is 0.21). - raise NotImplementedError + data = [{"price": "100", "quantity": "3"}] + result = calculate_revenue(data) + assert result[0]["revenue"] == 300.0 + assert result[0]["vat"] == 63.0 + def test_no_mutation(): # TODO: feed in a list, run any transform on it, assert the original # list is unchanged. This is the most important test in the file. - raise NotImplementedError + original = [{"product_name": "Laptop", "price": 999.99}] + copy = original.copy() + remove_invalid(original) + assert original == copy # original list is unchanged + diff --git a/task-2/AI_DEBUG.md b/task-2/AI_DEBUG.md index 83cad22..8c69d0d 100644 --- a/task-2/AI_DEBUG.md +++ b/task-2/AI_DEBUG.md @@ -12,12 +12,21 @@ What went wrong? Paste the traceback or the wrong-output sample. Include the fil ``` (paste here) +Python was not found; run without arguments to install from the Microsoft Store, or disable this shortcut from Settings > Apps > Advanced app settings > App execution aliases. +WARN: pip install failed; pipeline may fail with ModuleNotFoundError +Task 1 (Cleaner Pipeline): 10/60 — pipeline failed to run: Python was not found; run without arguments to install from the Microsoft Store, or disable this shortcut from Settings > Apps > Advanced app settings > App execution aliases. ``` ## The Prompt What did you ask the AI? Paste the actual prompt verbatim. (Include the code or stack trace you pasted alongside it; do NOT include any real `.env` values, API keys, or PII — replace those with ``.) +/Users/Gebruiker/c55-data-week-2/task-1/tests/test_transforms.py +Traceback (most recent call last): + File "c:\Users\Gebruiker\c55-data-week-2\task-1\tests\test_transforms.py", line 9, in + import pytest +ModuleNotFoundError: No module named 'pytest' +PS C:\Users\Gebruiker\ installed pytest but error still remains ``` (paste here) ``` @@ -28,6 +37,7 @@ What did the AI suggest? Did it work on the first try? Did you have to follow up ``` (paste here) +The problem is pytest is installed globally but not in your project's virtual environment. ``` ## Reflection @@ -35,3 +45,4 @@ What did the AI suggest? Did it work on the first try? Did you have to follow up Did you understand *why* the original code was broken before the AI told you? If not, what was the gap in your mental model? If you understood it before asking, why did you still ask the AI — speed, second opinion, or something else? (write here, ~100 words) +I discovered i have two pythons installed in my computer and the one needed was not the default path for execution \ No newline at end of file diff --git a/task-3/assets/azure_blob_week2.png b/task-3/assets/azure_blob_week2.png new file mode 100644 index 0000000..bec4bcc Binary files /dev/null and b/task-3/assets/azure_blob_week2.png differ diff --git a/task-3/assets/blob_url.txt b/task-3/assets/blob_url.txt new file mode 100644 index 0000000..de68e3c --- /dev/null +++ b/task-3/assets/blob_url.txt @@ -0,0 +1 @@ +https://sthyfstudentsdemo.blob.core.windows.net/week2-hannahwn/clean_sales.csv \ No newline at end of file