init
This commit is contained in:
illustris
35
scenario6-usdt-probes/Makefile
Normal file
35
scenario6-usdt-probes/Makefile
Normal file
@@ -0,0 +1,35 @@
|
||||
CC = gcc
|
||||
CFLAGS = -O2 -g -Wall
|
||||
|
||||
# Check if sys/sdt.h is available
|
||||
SDT_CHECK := $(shell echo '\#include <sys/sdt.h>' | $(CC) -E - >/dev/null 2>&1 && echo yes)
|
||||
|
||||
ifeq ($(SDT_CHECK),yes)
|
||||
CFLAGS += -DHAVE_SDT
|
||||
SDT_STATUS = "USDT probes ENABLED"
|
||||
else
|
||||
SDT_STATUS = "USDT probes DISABLED (install systemtap-sdt-dev)"
|
||||
endif
|
||||
|
||||
all: server
|
||||
@echo $(SDT_STATUS)
|
||||
|
||||
server: server.c
|
||||
$(CC) $(CFLAGS) -o $@ $<
|
||||
|
||||
# List the probes embedded in the binary
|
||||
list-probes: server
|
||||
@echo "Probes in binary (requires USDT support):"
|
||||
readelf -n server 2>/dev/null | grep -A2 "stapsdt" || echo "No probes found"
|
||||
|
||||
# Alternative: use perf to list probes
|
||||
perf-list: server
|
||||
perf probe -x ./server --list 2>/dev/null || echo "No probes found or perf not supported"
|
||||
|
||||
clean:
|
||||
rm -f server
|
||||
|
||||
install-deps:
|
||||
sudo apt install systemtap-sdt-dev bpftrace
|
||||
|
||||
.PHONY: all clean list-probes perf-list install-deps
|
||||
184
scenario6-usdt-probes/README.md
Normal file
184
scenario6-usdt-probes/README.md
Normal file
@@ -0,0 +1,184 @@
|
||||
# Scenario 6: USDT Probes - Custom Instrumentation
|
||||
|
||||
## Learning Objectives
|
||||
- Understand the difference between static and dynamic probes
|
||||
- Add USDT probes to C code
|
||||
- Trace probes with bpftrace
|
||||
- See how probes enable production debugging
|
||||
|
||||
## Background
|
||||
|
||||
**Dynamic probes** (like `perf probe`): Added at runtime, can break on any function
|
||||
**Static probes** (USDT): Compiled into the binary, designed by the developer
|
||||
|
||||
USDT probes are:
|
||||
- Nearly zero overhead when not traced
|
||||
- Stable API for debuggers/tracers
|
||||
- Self-documenting: probe names describe what's happening
|
||||
- Used by Python, Ruby, MySQL, PostgreSQL, and many other projects
|
||||
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
# Install USDT support and bpftrace
|
||||
sudo apt install systemtap-sdt-dev bpftrace
|
||||
```
|
||||
|
||||
## Files
|
||||
- `server.c` - Simulated server with USDT probes at key points
|
||||
|
||||
## Exercise 1: Build and Run
|
||||
|
||||
```bash
|
||||
make
|
||||
./server 3 10
|
||||
```
|
||||
|
||||
This processes 3 batches of 10 requests each.
|
||||
|
||||
## Exercise 2: List the Probes
|
||||
|
||||
```bash
|
||||
# Using readelf
|
||||
readelf -n ./server | grep -A2 stapsdt
|
||||
|
||||
# Or using perf
|
||||
sudo perf probe -x ./server --list
|
||||
```
|
||||
|
||||
You should see probes like:
|
||||
- `myserver:server_start`
|
||||
- `myserver:request_start`
|
||||
- `myserver:request_end`
|
||||
- `myserver:request_error`
|
||||
|
||||
## Exercise 3: Trace with bpftrace
|
||||
|
||||
### Count requests by type
|
||||
```bash
|
||||
# In terminal 1:
|
||||
sudo bpftrace -e '
|
||||
usdt:./server:myserver:request_start {
|
||||
@types[arg1] = count();
|
||||
}
|
||||
END {
|
||||
print(@types);
|
||||
}
|
||||
'
|
||||
|
||||
# In terminal 2:
|
||||
./server 5 50
|
||||
```
|
||||
|
||||
### Measure request latency
|
||||
```bash
|
||||
sudo bpftrace -e '
|
||||
usdt:./server:myserver:request_start {
|
||||
@start[arg0] = nsecs;
|
||||
}
|
||||
usdt:./server:myserver:request_end {
|
||||
$latency = (nsecs - @start[arg0]) / 1000000;
|
||||
@latency_ms = hist($latency);
|
||||
delete(@start[arg0]);
|
||||
}
|
||||
END {
|
||||
print(@latency_ms);
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
### Track errors
|
||||
```bash
|
||||
sudo bpftrace -e '
|
||||
usdt:./server:myserver:request_error {
|
||||
printf("ERROR: request %d failed with code %d\n", arg0, arg1);
|
||||
@errors = count();
|
||||
}
|
||||
'
|
||||
```
|
||||
|
||||
## Exercise 4: Trace with perf
|
||||
|
||||
```bash
|
||||
# Add probe
|
||||
sudo perf probe -x ./server 'myserver:request_start'
|
||||
|
||||
# Record
|
||||
sudo perf record -e 'probe_server:*' ./server 3 20
|
||||
|
||||
# Report
|
||||
sudo perf report
|
||||
```
|
||||
|
||||
## How USDT Probes Work
|
||||
|
||||
The `DTRACE_PROBE` macro inserts a NOP instruction:
|
||||
|
||||
```c
|
||||
DTRACE_PROBE2(myserver, request_start, req->id, req->type);
|
||||
```
|
||||
|
||||
Compiles to something like:
|
||||
```asm
|
||||
nop ; placeholder for probe
|
||||
```
|
||||
|
||||
When you activate tracing:
|
||||
1. Tracer finds the probe location (stored in ELF notes)
|
||||
2. Replaces NOP with a trap instruction (INT3 on x86)
|
||||
3. Trap triggers, tracer runs, returns control
|
||||
4. When tracing stops, NOP is restored
|
||||
|
||||
**Overhead when not tracing**: ~0 (just a NOP)
|
||||
**Overhead when tracing**: trap + handler execution
|
||||
|
||||
## Real-World Uses
|
||||
|
||||
### Python has USDT probes:
|
||||
```bash
|
||||
# If Python was built with --enable-dtrace
|
||||
sudo bpftrace -e 'usdt:/usr/bin/python3:python:function__entry { printf("%s\n", str(arg1)); }'
|
||||
```
|
||||
|
||||
### MySQL probes for query tracking:
|
||||
```bash
|
||||
sudo bpftrace -e 'usdt:/usr/sbin/mysqld:mysql:query__start { printf("%s\n", str(arg0)); }'
|
||||
```
|
||||
|
||||
## Discussion Questions
|
||||
|
||||
1. **When would you use USDT vs dynamic probes?**
|
||||
- USDT: Known important points, stable interface
|
||||
- Dynamic: Ad-hoc debugging, no source changes
|
||||
|
||||
2. **What's the trade-off of adding probes?**
|
||||
- Pro: Always available for debugging
|
||||
- Con: Must plan ahead, adds code complexity
|
||||
|
||||
3. **Why not just use printf debugging?**
|
||||
- Printf has overhead even when you don't need it
|
||||
- USDT has zero overhead until activated
|
||||
- USDT can be traced without rebuilding
|
||||
|
||||
## Advanced: Creating Custom Probes
|
||||
|
||||
The probe macros from `<sys/sdt.h>`:
|
||||
|
||||
```c
|
||||
DTRACE_PROBE(provider, name) // No arguments
|
||||
DTRACE_PROBE1(provider, name, arg1) // 1 argument
|
||||
DTRACE_PROBE2(provider, name, arg1, arg2) // 2 arguments
|
||||
// ... up to DTRACE_PROBE12
|
||||
```
|
||||
|
||||
Arguments can be integers or pointers. Strings need special handling.
|
||||
|
||||
## Key Takeaways
|
||||
|
||||
1. **USDT probes are designed-in observability**
|
||||
2. **Zero overhead when not actively tracing**
|
||||
3. **bpftrace makes probe usage easy**
|
||||
4. **Many production systems already have probes (Python, databases, etc.)**
|
||||
|
||||
This is an advanced topic - the main takeaway for beginners is that
|
||||
such instrumentation exists and enables powerful production debugging.
|
||||
141
scenario6-usdt-probes/server.c
Normal file
141
scenario6-usdt-probes/server.c
Normal file
@@ -0,0 +1,141 @@
|
||||
/*
|
||||
* Scenario 6: USDT Probes - Custom Instrumentation
|
||||
* =================================================
|
||||
* This program demonstrates User Statically Defined Tracepoints (USDT).
|
||||
* USDT probes allow you to add custom tracing points to your code
|
||||
* that have near-zero overhead when not actively traced.
|
||||
*
|
||||
* Compile:
|
||||
* gcc -O2 -g -o server server.c
|
||||
*
|
||||
* The probes use sys/sdt.h which is provided by systemtap-sdt-dev on Ubuntu.
|
||||
* Install: sudo apt install systemtap-sdt-dev
|
||||
*
|
||||
* EXERCISES:
|
||||
* 1. Run normally: ./server
|
||||
* 2. List probes: perf probe -x ./server --list
|
||||
* 3. Trace probes: See README.md for bpftrace examples
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <string.h>
|
||||
|
||||
/*
|
||||
* USDT probe macros from systemtap-sdt-dev
|
||||
* If not available, we define dummy macros
|
||||
*/
|
||||
#ifdef HAVE_SDT
|
||||
#include <sys/sdt.h>
|
||||
#else
|
||||
/* Dummy macros when SDT isn't available */
|
||||
#define DTRACE_PROBE(provider, name)
|
||||
#define DTRACE_PROBE1(provider, name, arg1)
|
||||
#define DTRACE_PROBE2(provider, name, arg1, arg2)
|
||||
#define DTRACE_PROBE3(provider, name, arg1, arg2, arg3)
|
||||
#endif
|
||||
|
||||
/* Simulated request structure */
|
||||
struct request {
|
||||
int id;
|
||||
int type; /* 0=read, 1=write, 2=delete */
|
||||
int size; /* payload size */
|
||||
char data[64];
|
||||
};
|
||||
|
||||
/* Statistics */
|
||||
static unsigned long total_requests = 0;
|
||||
static unsigned long total_errors = 0;
|
||||
|
||||
/*
|
||||
* Process a single request
|
||||
* We add USDT probes at entry, exit, and error points
|
||||
*/
|
||||
int process_request(struct request *req) {
|
||||
/* Probe: request processing started */
|
||||
DTRACE_PROBE2(myserver, request_start, req->id, req->type);
|
||||
|
||||
total_requests++;
|
||||
|
||||
/* Simulate variable processing time based on request type */
|
||||
int delay_ms;
|
||||
switch (req->type) {
|
||||
case 0: delay_ms = 10 + (rand() % 20); break; /* read: 10-30ms */
|
||||
case 1: delay_ms = 50 + (rand() % 50); break; /* write: 50-100ms */
|
||||
case 2: delay_ms = 5 + (rand() % 10); break; /* delete: 5-15ms */
|
||||
default: delay_ms = 100;
|
||||
}
|
||||
|
||||
/* Simulate some work */
|
||||
usleep(delay_ms * 1000);
|
||||
|
||||
/* Simulate occasional errors (10% chance) */
|
||||
if (rand() % 10 == 0) {
|
||||
DTRACE_PROBE2(myserver, request_error, req->id, -1);
|
||||
total_errors++;
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Probe: request completed successfully */
|
||||
DTRACE_PROBE3(myserver, request_end, req->id, req->type, delay_ms);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Batch processing function
|
||||
*/
|
||||
void process_batch(int batch_id, int count) {
|
||||
DTRACE_PROBE2(myserver, batch_start, batch_id, count);
|
||||
|
||||
printf("Processing batch %d with %d requests...\n", batch_id, count);
|
||||
|
||||
struct request req;
|
||||
for (int i = 0; i < count; i++) {
|
||||
req.id = batch_id * 1000 + i;
|
||||
req.type = rand() % 3;
|
||||
req.size = 64;
|
||||
snprintf(req.data, sizeof(req.data), "request_%d", req.id);
|
||||
|
||||
process_request(&req);
|
||||
}
|
||||
|
||||
DTRACE_PROBE1(myserver, batch_end, batch_id);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
int num_batches = 5;
|
||||
int requests_per_batch = 20;
|
||||
|
||||
if (argc > 1) num_batches = atoi(argv[1]);
|
||||
if (argc > 2) requests_per_batch = atoi(argv[2]);
|
||||
|
||||
srand(time(NULL));
|
||||
|
||||
printf("USDT Probe Demo Server\n");
|
||||
printf("======================\n");
|
||||
printf("Batches: %d, Requests per batch: %d\n\n", num_batches, requests_per_batch);
|
||||
|
||||
#ifndef HAVE_SDT
|
||||
printf("Note: Compiled without USDT support.\n");
|
||||
printf("To enable probes: sudo apt install systemtap-sdt-dev\n");
|
||||
printf("Then compile with: gcc -DHAVE_SDT -O2 -g -o server server.c\n\n");
|
||||
#endif
|
||||
|
||||
DTRACE_PROBE(myserver, server_start);
|
||||
|
||||
for (int i = 0; i < num_batches; i++) {
|
||||
process_batch(i, requests_per_batch);
|
||||
}
|
||||
|
||||
DTRACE_PROBE(myserver, server_stop);
|
||||
|
||||
printf("\n=== Summary ===\n");
|
||||
printf("Total requests: %lu\n", total_requests);
|
||||
printf("Total errors: %lu (%.1f%%)\n",
|
||||
total_errors, 100.0 * total_errors / total_requests);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user