diff --git a/.gitignore b/.gitignore index 339a448..3e36f93 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,5 @@ venv/ .venv/ task-*/venv/ task-*/.venv/ +__pycache__/ +*.pyc diff --git a/task-1/output/clean_sales.csv b/task-1/output/clean_sales.csv new file mode 100644 index 0000000..4e444dc --- /dev/null +++ b/task-1/output/clean_sales.csv @@ -0,0 +1,13 @@ +transaction_id,product_name,category,price,quantity,customer_email,date,revenue,vat +1,Laptop Pro,Electronics,999.99,2,alice@example.com,2024-03-15,1999.98,420.0 +2,Wireless Mouse,Electronics,29.99,5,bob@company.com,2024-03-15,149.95,31.49 +3,Usb Cable,Electronics,4.99,10,,2024-03-16,49.9,10.48 +4,Office Chair,Furniture,349.5,1,charlie@work.org,2024-03-16,349.5,73.39 +5,Standing Desk,Furniture,599.0,1,charlie@work.org,not_a_date,599.0,125.79 +9,Webcam Hd,Electronics,54.99,1,,2024-03-18,54.99,11.55 +10,Desk Lamp,Furniture,34.99,4,grace@university.edu,2024-03-19,139.96,29.39 +11,Noise Cancelling Headphones,Electronics,199.99,1,alice@example.com,2024-03-19,199.99,42.0 +12,Cable Management Kit,Furniture,15.99,6,henry@business.com,2024-03-20,95.94,20.15 +13,Ergonomic Mouse Pad,Furniture,24.99,3,ivan@email.com,2024-03-20,74.97,15.74 +14,Laptop Stand,Furniture,45.99,2,jenny@work.org,2024-03-21,91.98,19.32 +15,Bluetooth Speaker,Unknown,39.99,1,karl@startup.io,2024-03-21,39.99,8.4 diff --git a/task-1/src/config.py b/task-1/src/config.py index e59f7c1..17479ea 100644 --- a/task-1/src/config.py +++ b/task-1/src/config.py @@ -20,16 +20,13 @@ def _required(name: str) -> str: + """Read an env var; fail loudly if missing.""" - # TODO 2: Read os.environ[name]; if not set, raise ValueError with a - # message that names the missing variable AND points at .env.example. - raise NotImplementedError("Implement _required: see TODO 2 in config.py") - - -# TODO 3: Replace the placeholder lines below by calling _required(...) for -# each variable. INPUT_PATH and OUTPUT_PATH must be importable from this -# module by the rest of the pipeline as a relative import -# (`from .config import INPUT_PATH, ...`), since the pipeline runs as -# `python -m src.pipeline`. -INPUT_PATH: str = "" # TODO: _required("INPUT_PATH") -OUTPUT_PATH: str = "" # TODO: _required("OUTPUT_PATH") + value = os.environ.get(name) + if not value: + raise ValueError(f"Missing {name}.") + + return value + +INPUT_PATH: str = _required("INPUT_PATH") +OUTPUT_PATH: str = _required("OUTPUT_PATH") diff --git a/task-1/src/models.py b/task-1/src/models.py index 865b2b1..e23da40 100644 --- a/task-1/src/models.py +++ b/task-1/src/models.py @@ -10,24 +10,22 @@ """ from dataclasses import dataclass - -# TODO 1: Define a @dataclass called Transaction with these fields: -# transaction_id: int -# product_name: str -# category: str -# price: float -# quantity: int -# customer_email: str -# date: str -# revenue: float = 0.0 -# vat: float = 0.0 -# -# TODO 2: Add __post_init__ that raises ValueError when: -# - self.price < 0 (with a message naming the bad value) -# - not self.product_name.strip() (empty / whitespace-only product name) - - -# Replace this stub with your dataclass: @dataclass class Transaction: - transaction_id: int # TODO: replace this stub with the full field list above + transaction_id: int + product_name: str + category: str + price: float + quantity: int + customer_email: str + date: str + revenue: float = 0.0 + vat: float = 0.0 + + def __post_init__(self): + if self.price < 0: + raise ValueError(f"Price cannot be negative: {self.price}") + + if not self.product_name.strip(): + raise ValueError("Product name cannot be empty") + diff --git a/task-1/src/pipeline.py b/task-1/src/pipeline.py index adce136..7e83600 100644 --- a/task-1/src/pipeline.py +++ b/task-1/src/pipeline.py @@ -31,31 +31,45 @@ def read_csv(path: str) -> list[dict]: """Read a CSV file into a list of dicts. I/O only — no business rules.""" - # TODO: implement using csv.DictReader. - raise NotImplementedError + with open(path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + return list(reader) + def write_csv(rows: list[dict], path: str) -> None: """Write a list of dicts to CSV. I/O only — no business rules.""" - # TODO: implement using csv.DictWriter. - raise NotImplementedError + if not rows: + return + + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(rows) def run() -> None: + raw = read_csv(INPUT_PATH) data = remove_invalid(raw) data = clean_fields(data) data = filter_zero_quantity(data) data = calculate_revenue(data) + transactions = [ + Transaction( + transaction_id=int(row["transaction_id"]), + product_name=row["product_name"], + category=row["category"], + price=float(row["price"]), + quantity=int(row["quantity"]), + customer_email=row["customer_email"], + date=row["date"], + revenue=float(row["revenue"]), + vat=float(row["vat"]), + ) + for row in data +] - # Materialise as Transaction instances so the dataclass __post_init__ - # acts as a final guard before serialisation. - # TODO: cast price / quantity / revenue / vat to the right types here - # if your transforms left them as strings, then iterate over `data` - # to build Transaction(**row) for each cleaned row. - transactions = [Transaction(**row) for row in data] - - # Output dir must exist. Use pathlib for cross-platform safety. Path(OUTPUT_PATH).parent.mkdir(parents=True, exist_ok=True) write_csv([asdict(t) for t in transactions], OUTPUT_PATH) diff --git a/task-1/src/transforms.py b/task-1/src/transforms.py index e6bdd76..61c97d6 100644 --- a/task-1/src/transforms.py +++ b/task-1/src/transforms.py @@ -16,10 +16,27 @@ def remove_invalid(rows: list[dict]) -> list[dict]: """Remove rows with empty product_name OR negative price. + Empty here means missing, "", or whitespace-only. """ - # TODO: implement. Return a new list, do not mutate `rows`. - raise NotImplementedError + clean_rows = [] + for row in rows: + product_name = row.get("product_name", "") + price = row.get("price", 0) + + if not product_name.strip(): + continue # Skip rows with empty product_name + + try: + price_value = float(price) + if price_value < 0: + continue # Skip rows with negative price + except (ValueError, TypeError): + continue # Skip rows where price is not a valid number + + clean_rows.append(row.copy()) # Keep valid rows + + return clean_rows def clean_fields(rows: list[dict]) -> list[dict]: @@ -31,20 +48,55 @@ def clean_fields(rows: list[dict]) -> list[dict]: Return a new list. Do not mutate the input rows. """ - # TODO: implement. - raise NotImplementedError + clean_rows = [] + for row in rows: + clean_row = row.copy() # Create a copy to avoid mutating the input + # Clean product_name + product_name = clean_row.get("product_name", "") + clean_row["product_name"] = product_name.strip().title() + + # Clean customer_email + customer_email = clean_row.get("customer_email", "") + clean_row["customer_email"] = customer_email.strip().lower() + + # Clean category + category = clean_row.get("category", "") + clean_row["category"] = category.strip().title() if category.strip() else "Unknown" + + clean_rows.append(clean_row) + return clean_rows def calculate_revenue(rows: list[dict], vat_rate: float = 0.21) -> list[dict]: """Add 'revenue' (price * quantity) and 'vat' (revenue * vat_rate) fields. Round both to 2 decimal places. Coerce price/quantity from string if needed. """ - # TODO: implement. - raise NotImplementedError + clean_rows = [] + for row in rows: + clean_row = row.copy() # Create a copy to avoid mutating the input + + try: + price = float(clean_row.get("price", 0)) + quantity = int(clean_row.get("quantity", 0)) + revenue = round(price * quantity, 2) + vat = round(revenue * vat_rate, 2) + clean_row["price"] = price + clean_row["quantity"] = quantity + clean_row["revenue"] = revenue + clean_row["vat"] = vat + except (ValueError, TypeError): + continue # Skip rows where price or quantity is not a valid number def filter_zero_quantity(rows: list[dict]) -> list[dict]: """Remove rows where quantity is 0.""" - # TODO: implement. - raise NotImplementedError + clean_rows = [] + for row in rows: + try: + quantity = int(row.get("quantity", 0)) + if quantity != 0: + clean_rows.append(row.copy()) # Keep rows where quantity is not 0 + except (ValueError, TypeError): + continue # Skip rows where quantity is not a valid number + return clean_rows diff --git a/task-1/tests/__pycache__/__init__.cpython-311.pyc b/task-1/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..3872fe9 Binary files /dev/null and b/task-1/tests/__pycache__/__init__.cpython-311.pyc differ diff --git a/task-1/tests/__pycache__/test_transforms.cpython-311-pytest-9.0.3.pyc b/task-1/tests/__pycache__/test_transforms.cpython-311-pytest-9.0.3.pyc new file mode 100644 index 0000000..8e045f3 Binary files /dev/null and b/task-1/tests/__pycache__/test_transforms.cpython-311-pytest-9.0.3.pyc differ diff --git a/task-1/tests/test_transforms.py b/task-1/tests/test_transforms.py index 8d6bd80..5ffb7da 100644 --- a/task-1/tests/test_transforms.py +++ b/task-1/tests/test_transforms.py @@ -17,24 +17,50 @@ def test_remove_invalid_drops_empty_names(): - # TODO: feed in 3 rows (one valid, one empty product_name, one - # whitespace-only product_name); assert only the valid one survives. - raise NotImplementedError + data = [ + {"product_name": "Laptop", "price": 999.99}, + {"product_name": "", "price": 50.0}, + {"product_name": " ", "price": 25.0}, + ] + result = remove_invalid(data) + assert len(result) == 1 + assert result[0]["product_name"] == "Laptop" + assert result[0]["price"] == 999.99 def test_clean_fields_normalizes_names(): - # TODO: feed a row with messy product_name and uppercase email; assert - # the output has stripped + title-cased name and lowercase email. - raise NotImplementedError + data = [ + { + "product_name": " laptop stand ", + "customer_email": "ZEUS@OLYMPUS.ORG", + "category": ""} + ] + result = clean_fields(data) + assert result[0]["product_name"] == "Laptop Stand" + assert result[0]["customer_email"] == "zeus@olympus.org" + assert result[0]["category"] == "Unknown" def test_calculate_revenue_adds_fields(): - # TODO: feed a row with price=100, quantity=3; assert output has - # revenue=300.0 and vat=63.0 (default VAT rate is 0.21). - raise NotImplementedError - + data =[ + { + "product_name": "Laptop", + "price": "100", + "quantity": "3" + } + ] + result = calculate_revenue(data) + assert result[0]["revenue"] == 300.0 + assert result[0]["vat"] == 63.0 def test_no_mutation(): - # TODO: feed in a list, run any transform on it, assert the original - # list is unchanged. This is the most important test in the file. - raise NotImplementedError + data = [ + { + "product_name": " laptop ", + "customer_email":" ZEUS@Test.coM", + "category": "",} + ] + + original_data = [row.copy() for row in data] + clean_fields(data) + assert data == original_data diff --git a/task-2/AI_DEBUG.md b/task-2/AI_DEBUG.md index 83cad22..4cc3e9a 100644 --- a/task-2/AI_DEBUG.md +++ b/task-2/AI_DEBUG.md @@ -10,28 +10,47 @@ Aim for 100-200 words per section. Bullet points are fine. What went wrong? Paste the traceback or the wrong-output sample. Include the file and the line you were running when it broke. +While implementing the `write_csv()` function in `pipeline.py`, I forgot to open the file in write mode. My original code used: + +```python +with open(path, newline="", encoding="utf-8") as f: ``` -(paste here) -``` + +Because of this Python opened the file in read mode by default. When the pipeline tried to write rows using csv.DictWriter, it failed because the file was not writable. ## The Prompt What did you ask the AI? Paste the actual prompt verbatim. (Include the code or stack trace you pasted alongside it; do NOT include any real `.env` values, API keys, or PII — replace those with ``.) -``` -(paste here) +## The Prompt + +I asked ChatGPT: + +I think there might be something wrong with my write_cs() function. I am using csv.DictWriter correctly, but I am not sure if open() allows writing. + +```python +def write_csv(rows: list[dict], path: str) -> None: + with open(path, newline="", encoding="utf-8") as f: + if not rows: + return + + writer = csv.DictWriter(f, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(rows) ``` ## The Solution What did the AI suggest? Did it work on the first try? Did you have to follow up? Paste the final code change (a small diff is best). -``` -(paste here) +ChatGPT pointed out that I forgot to add "w" mode to the open() call. Python opens files in read mode by default, so the file could not be written to. The AI suggested changing the code to: + +```python + with open(path, "w", newline="", encoding="utf-8") as f: ``` ## Reflection -Did you understand *why* the original code was broken before the AI told you? If not, what was the gap in your mental model? If you understood it before asking, why did you still ask the AI — speed, second opinion, or something else? +Did you understand _why_ the original code was broken before the AI told you? If not, what was the gap in your mental model? If you understood it before asking, why did you still ask the AI — speed, second opinion, or something else? -(write here, ~100 words) +understood that write_csv() needed to create and save a CSV file, but I forgot that open() defaults to read mode if no mode is provided, because I focused more on the csv.DictWriter logic than on the file opening step itself. I asked the AI because I wanted a quick review before running the pipeline, and it helped catch a small but important mistake immediately. This reminded me that many bugs come from defaults and small details rather than complex logic diff --git a/task-3/assets/azure_blob_week2.png b/task-3/assets/azure_blob_week2.png new file mode 100644 index 0000000..fab48f9 Binary files /dev/null and b/task-3/assets/azure_blob_week2.png differ diff --git a/task-3/assets/blob_url.txt b/task-3/assets/blob_url.txt new file mode 100644 index 0000000..1e581c0 --- /dev/null +++ b/task-3/assets/blob_url.txt @@ -0,0 +1 @@ +https://sthyfstudentsdemo.blob.core.windows.net/week2-mohammedalfakih-dev/clean_sales.csv \ No newline at end of file