Pony-Alpha-2-Dataset-Training/skills/skill-api-engineering.md

API Engineering Expert Skill

Activation Criteria

Activate this skill when the user:

• Designs RESTful APIs or GraphQL schemas
• Implements gRPC services
• Designs API versioning strategies
• Implements authentication (JWT, OAuth2, API keys)
• Creates rate limiting and throttling mechanisms
• Designs pagination and filtering systems
• Implements error handling standards
• Creates OpenAPI/Swagger specifications
• Designs API SDKs
• Implements API gateway patterns
• Builds real-time APIs (WebSocket, SSE, webhooks)
• Designs microservice communication
• Creates API documentation
• Implements API testing strategies
• Designs API security and authorization

Core Methodology

1. REST API Design

RESTful API Best Practices

// TypeScript REST API Implementation
// Complete API with best practices

import express, { Request, Response, NextFunction } from 'express';
import { z } from 'zod';
import helmet from 'helmet';
import rateLimit from 'express-rate-limit';
import slowDown from 'express-slow-down';
import cors from 'cors';
import compression from 'compression';
import morgan from 'morgan';
import { body, query, param, validationResult } from 'express-validator';

// Application Configuration
const config = {
  port: process.env.PORT || 3000,
  env: process.env.NODE_ENV || 'development',
  apiVersion: 'v1',
  rateLimitWindow: 15 * 60 * 1000, // 15 minutes
  rateLimitMax: 100, // 100 requests per window
};

// Request Context Interface
interface RequestContext {
  requestId: string;
  userId?: string;
  apiKey?: string;
  userAgent: string;
  ip: string;
  timestamp: Date;
}

// Extend Express Request
interface AuthenticatedRequest extends Request {
  context: RequestContext;
  user?: {
    id: string;
    email: string;
    role: string;
  };
}

// API Response Standardization
class APIResponse<T> {
  constructor(
    public success: boolean,
    public data?: T,
    public error?: ErrorResponse,
    public meta?: ResponseMeta
  ) {}

  static ok<T>(data: T, meta?: ResponseMeta): APIResponse<T> {
    return new APIResponse(true, data, undefined, meta);
  }

  static error(error: ErrorResponse): APIResponse<null> {
    return new APIResponse(false, undefined, error);
  }

  static paginated<T>(data: T[], pagination: PaginationMeta): APIResponse<T[]> {
    return new APIResponse(true, data, undefined, { pagination });
  }
}

interface ErrorResponse {
  code: string;
  message: string;
  details?: Record<string, any>;
  stack?: string;
}

interface ResponseMeta {
  pagination?: PaginationMeta;
  requestId?: string;
  timestamp?: string;
}

interface PaginationMeta {
  page: number;
  limit: number;
  total: number;
  totalPages: number;
  hasNext: boolean;
  hasPrevious: boolean;
}

// Error Handler
class APIError extends Error {
  constructor(
    public statusCode: number,
    public code: string,
    message: string,
    public details?: Record<string, any>
  ) {
    super(message);
    this.name = 'APIError';
  }
}

// Validation Schemas using Zod
const createUserSchema = z.object({
  email: z.string().email('Invalid email format'),
  password: z.string().min(8, 'Password must be at least 8 characters'),
  firstName: z.string().min(1, 'First name is required'),
  lastName: z.string().min(1, 'Last name is required'),
  role: z.enum(['user', 'admin', 'moderator']).default('user'),
});

const updateUserSchema = z.object({
  email: z.string().email().optional(),
  firstName: z.string().min(1).optional(),
  lastName: z.string().min(1).optional(),
  role: z.enum(['user', 'admin', 'moderator']).optional(),
});

const queryUserSchema = z.object({
  page: z.coerce.number().int().positive().default(1),
  limit: z.coerce.number().int().positive().max(100).default(20),
  sort: z.enum(['createdAt', 'email', 'firstName', 'lastName']).default('createdAt'),
  order: z.enum(['asc', 'desc']).default('asc'),
  search: z.string().optional(),
  role: z.enum(['user', 'admin', 'moderator']).optional(),
});

// Express Application Setup
const app = express();

// Security Middleware
app.use(helmet());
app.use(compression());

// CORS Configuration
app.use(cors({
  origin: (origin, callback) => {
    const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(',') || ['http://localhost:3000'];
    if (!origin || allowedOrigins.includes(origin)) {
      callback(null, true);
    } else {
      callback(new Error('Not allowed by CORS'));
    }
  },
  credentials: true,
  methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'],
  allowedHeaders: ['Content-Type', 'Authorization', 'X-Request-ID'],
}));

// Rate Limiting
const limiter = rateLimit({
  windowMs: config.rateLimitWindow,
  max: config.rateLimitMax,
  message: {
    error: {
      code: 'RATE_LIMIT_EXCEEDED',
      message: 'Too many requests, please try again later.',
    },
  },
  standardHeaders: true,
  legacyHeaders: false,
  keyGenerator: (req: AuthenticatedRequest) => {
    return req.user?.id || req.ip;
  },
});

// Slow Down (gradually slow down responses)
const speedLimiter = slowDown({
  windowMs: config.rateLimitWindow,
  delayAfter: 50,
  delayMs: 500,
  keyGenerator: (req: AuthenticatedRequest) => {
    return req.user?.id || req.ip;
  },
});

// Request Logging
if (config.env === 'development') {
  app.use(morgan('dev'));
} else {
  app.use(morgan('combined'));
}

// Body Parser
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true, limit: '10mb' }));

// Request ID Middleware
app.use((req: AuthenticatedRequest, res: Response, next: NextFunction) => {
  req.context = {
    requestId: req.headers['x-request-id'] as string || generateRequestId(),
    userAgent: req.headers['user-agent'] || '',
    ip: req.ip,
    timestamp: new Date(),
  };
  res.setHeader('X-Request-ID', req.context.requestId);
  next();
});

// Authentication Middleware
const authenticate = async (req: AuthenticatedRequest, res: Response, next: NextFunction) => {
  try {
    const token = req.headers.authorization?.replace('Bearer ', '');

    if (!token) {
      throw new APIError(401, 'UNAUTHORIZED', 'Authentication required');
    }

    // Verify JWT token
    const decoded = await verifyJWT(token);
    req.user = decoded;

    next();
  } catch (error) {
    next(error);
  }
};

// Authorization Middleware
const authorize = (...roles: string[]) => {
  return (req: AuthenticatedRequest, res: Response, next: NextFunction) => {
    if (!req.user) {
      throw new APIError(401, 'UNAUTHORIZED', 'Authentication required');
    }

    if (!roles.includes(req.user.role)) {
      throw new APIError(403, 'FORBIDDEN', 'Insufficient permissions');
    }

    next();
  };
};

// Validation Middleware
const validate = (schema: z.ZodSchema) => {
  return (req: AuthenticatedRequest, res: Response, next: NextFunction) => {
    try {
      req.body = schema.parse(req.body);
      next();
    } catch (error) {
      if (error instanceof z.ZodError) {
        throw new APIError(400, 'VALIDATION_ERROR', 'Validation failed', {
          errors: error.errors,
        });
      }
      next(error);
    }
  };
};

const validateQuery = (schema: z.ZodSchema) => {
  return (req: AuthenticatedRequest, res: Response, next: NextFunction) => {
    try {
      req.query = schema.parse(req.query);
      next();
    } catch (error) {
      if (error instanceof z.ZodError) {
        throw new APIError(400, 'VALIDATION_ERROR', 'Query validation failed', {
          errors: error.errors,
        });
      }
      next(error);
    }
  };
};

// User Controller
class UserController {
  // GET /api/v1/users
  static async list(req: AuthenticatedRequest, res: Response, next: NextFunction) {
    try {
      const query = req.query as z.infer<typeof queryUserSchema>;

      // Build filters
      const filters: any = {};
      if (query.search) {
        filters.$or = [
          { email: { $regex: query.search, $options: 'i' } },
          { firstName: { $regex: query.search, $options: 'i' } },
          { lastName: { $regex: query.search, $options: 'i' } },
        ];
      }
      if (query.role) {
        filters.role = query.role;
      }

      // Execute query with pagination
      const skip = (query.page - 1) * query.limit;
      const [users, total] = await Promise.all([
        User.find(filters)
          .sort({ [query.sort]: query.order === 'asc' ? 1 : -1 })
          .skip(skip)
          .limit(query.limit)
          .lean(),
        User.countDocuments(filters),
      ]);

      const totalPages = Math.ceil(total / query.limit);

      const response = APIResponse.paginated(users, {
        page: query.page,
        limit: query.limit,
        total,
        totalPages,
        hasNext: query.page < totalPages,
        hasPrevious: query.page > 1,
      });

      res.json(response);
    } catch (error) {
      next(error);
    }
  }

  // GET /api/v1/users/:id
  static async get(req: AuthenticatedRequest, res: Response, next: NextFunction) {
    try {
      const { id } = req.params;

      const user = await User.findById(id).lean();
      if (!user) {
        throw new APIError(404, 'USER_NOT_FOUND', 'User not found');
      }

      res.json(APIResponse.ok(user));
    } catch (error) {
      next(error);
    }
  }

  // POST /api/v1/users
  static async create(req: AuthenticatedRequest, res: Response, next: NextFunction) {
    try {
      const data = req.body as z.infer<typeof createUserSchema>;

      // Check if user exists
      const existing = await User.findOne({ email: data.email });
      if (existing) {
        throw new APIError(409, 'USER_EXISTS', 'User with this email already exists');
      }

      // Hash password
      const hashedPassword = await hashPassword(data.password);

      // Create user
      const user = await User.create({
        ...data,
        password: hashedPassword,
      });

      // Generate tokens
      const accessToken = await generateAccessToken(user);
      const refreshToken = await generateRefreshToken(user);

      res.status(201).json(
        APIResponse.ok({
          user: user.toJSON(),
          tokens: { accessToken, refreshToken },
        })
      );
    } catch (error) {
      next(error);
    }
  }

  // PATCH /api/v1/users/:id
  static async update(req: AuthenticatedRequest, res: Response, next: NextFunction) {
    try {
      const { id } = req.params;
      const data = req.body as z.infer<typeof updateUserSchema>;

      const user = await User.findById(id);
      if (!user) {
        throw new APIError(404, 'USER_NOT_FOUND', 'User not found');
      }

      // Check permissions
      if (req.user?.id !== id && req.user?.role !== 'admin') {
        throw new APIError(403, 'FORBIDDEN', 'You can only update your own profile');
      }

      // Update user
      Object.assign(user, data);
      await user.save();

      res.json(APIResponse.ok(user.toJSON()));
    } catch (error) {
      next(error);
    }
  }

  // DELETE /api/v1/users/:id
  static async delete(req: AuthenticatedRequest, res: Response, next: NextFunction) {
    try {
      const { id } = req.params;

      const user = await User.findById(id);
      if (!user) {
        throw new APIError(404, 'USER_NOT_FOUND', 'User not found');
      }

      // Check permissions (only admins can delete)
      if (req.user?.role !== 'admin') {
        throw new APIError(403, 'FORBIDDEN', 'Only admins can delete users');
      }

      await User.findByIdAndDelete(id);

      res.status(204).send();
    } catch (error) {
      next(error);
    }
  }
}

// API Routes
const apiRouter = express.Router();

// Public routes
apiRouter.post('/auth/register', validate(createUserSchema), UserController.create);
apiRouter.post('/auth/login', authenticate, ...);

// Protected routes
apiRouter.use(authenticate);
apiRouter.use(limiter);
apiRouter.use(speedLimiter);

// User routes
apiRouter.get('/users', authorize('admin'), validateQuery(queryUserSchema), UserController.list);
apiRouter.get('/users/:id', UserController.get);
apiRouter.patch('/users/:id', validate(updateUserSchema), UserController.update);
apiRouter.delete('/users/:id', authorize('admin'), UserController.delete);

// Mount API router
app.use(`/api/${config.apiVersion}`, apiRouter);

// Health check endpoint
app.get('/health', (req: Request, res: Response) => {
  res.json({
    status: 'ok',
    timestamp: new Date().toISOString(),
    uptime: process.uptime(),
    environment: config.env,
  });
});

// Error Handler
app.use((err: Error, req: AuthenticatedRequest, res: Response, next: NextFunction) => {
  if (err instanceof APIError) {
    res.status(err.statusCode).json(
      APIResponse.error({
        code: err.code,
        message: err.message,
        details: err.details,
        stack: config.env === 'development' ? err.stack : undefined,
      })
    );
  } else {
    res.status(500).json(
      APIResponse.error({
        code: 'INTERNAL_SERVER_ERROR',
        message: 'An unexpected error occurred',
        stack: config.env === 'development' ? err.stack : undefined,
      })
    );
  }
});

// 404 Handler
app.use((req: Request, res: Response) => {
  res.status(404).json(
    APIResponse.error({
      code: 'NOT_FOUND',
      message: 'Resource not found',
    })
  );
});

// Start Server
app.listen(config.port, () => {
  console.log(`API server listening on port ${config.port}`);
});

// Utility Functions
function generateRequestId(): string {
  return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}

async function verifyJWT(token: string): Promise<any> {
  // JWT verification implementation
  return {};
}

async function generateAccessToken(user: any): Promise<string> {
  // Access token generation implementation
  return '';
}

async function generateRefreshToken(user: any): Promise<string> {
  // Refresh token generation implementation
  return '';
}

async function hashPassword(password: string): Promise<string> {
  // Password hashing implementation
  return '';
}

// Database Model
class User {
  static async find(filters: any) { return []; }
  static async findById(id: string) { return null; }
  static async findOne(filters: any) { return null; }
  static async create(data: any) { return {}; }
  static async countDocuments(filters: any) { return 0; }
  static async findByIdAndUpdate(id: string, data: any) { return {}; }
  static async findByIdAndDelete(id: string) { return {}; }
}

2. GraphQL API Design

Production GraphQL Server

// GraphQL API Implementation
// Complete GraphQL server with best practices

import { ApolloServer } from '@apollo/server';
import { expressMiddleware } from '@apollo/server/express4';
import { ApolloServerPluginDrainHttpServer } from '@apollo/server/plugin/drainHttpServer';
import { makeExecutableSchema } from '@graphql-tools/schema';
import { WebSocketServer } from 'ws';
import { useServer } from 'graphql-ws/lib/use/ws';
import express from 'express';
import http from 'http';
import cors from 'cors';
import { JSONSchemaLoader } from '@graphql-tools/json-file-loader';
import { loadFilesSync } from '@graphql-tools/load-files';
import { mergeResolvers, mergeTypeDefs } from '@graphql-tools/merge';
import { IResolvers } from '@graphql-tools/utils';
import { GraphQLError } from 'graphql';

// Type Definitions
const typeDefs = /* GraphQL */ `
  scalar Date
  scalar Upload
  scalar JSON

  directive @auth(requires: [String!]!) on OBJECT | FIELD_DEFINITION
  directive @rateLimit(limit: Int!, duration: Int!) on FIELD_DEFINITION
  directive @cache(ttl: Int!) on FIELD_DEFINITION
  directive @deprecated(reason: String) on FIELD_DEFINITION | ENUM_VALUE

  type Query {
    me: User @auth(requires: ["USER"])
    user(id: ID!): User @cache(ttl: 300)
    users(
      first: Int
      after: String
      filter: UserFilter
      sort: UserSort
    ): UserConnection! @auth(requires: ["ADMIN"])
  }

  type Mutation {
    signUp(input: SignUpInput!): AuthPayload!
    signIn(input: SignInInput!): AuthPayload!
    refreshToken(input: RefreshTokenInput!): AuthPayload!
    updateUser(input: UpdateUserInput!): User! @auth(requires: ["USER"])
    deleteUser(id: ID!): Boolean! @auth(requires: ["ADMIN"])
    uploadAvatar(file: Upload!): String! @auth(requires: ["USER"])
  }

  type Subscription {
    userUpdated(userId: ID!): User!
    messageSent(chatId: ID!): Message!
  }

  type User {
    id: ID!
    email: String!
    firstName: String!
    lastName: String!
    fullName: String!
    avatar: String
    role: Role!
    createdAt: Date!
    updatedAt: Date!
    posts(first: Int, after: String): PostConnection!
  }

  type AuthPayload {
    accessToken: String!
    refreshToken: String!
    user: User!
  }

  type UserConnection {
    edges: [UserEdge!]!
    pageInfo: PageInfo!
    totalCount: Int!
  }

  type UserEdge {
    node: User!
    cursor: String!
  }

  type PageInfo {
    hasNextPage: Boolean!
    hasPreviousPage: Boolean!
    startCursor: String
    endCursor: String
  }

  input SignUpInput {
    email: String!
    password: String!
    firstName: String!
    lastName: String!
  }

  input SignInInput {
    email: String!
    password: String!
  }

  input RefreshTokenInput {
    token: String!
  }

  input UpdateUserInput {
    firstName: String
    lastName: String
    avatar: String
  }

  input UserFilter {
    search: String
    role: Role
  }

  input UserSort {
    field: UserSortField!
    direction: SortDirection!
  }

  enum UserSortField {
    CREATED_AT
    EMAIL
    FIRST_NAME
    LAST_NAME
  }

  enum SortDirection {
    ASC
    DESC
  }

  enum Role {
    USER
    ADMIN
    MODERATOR
  }
`;

// Resolvers
const resolvers: IResolvers = {
  Query: {
    me: async (_root, _args, { user, dataSources }) => {
      if (!user) {
        throw new GraphQLError('Not authenticated', {
          extensions: { code: 'UNAUTHENTICATED' },
        });
      }
      return dataSources.userAPI.getUser(user.id);
    },

    user: async (_root, { id }, { dataSources, cacheControl }) => {
      cacheControl.setCacheHint({ ttl: 300 });
      return dataSources.userAPI.getUser(id);
    },

    users: async (_root, { first = 20, after, filter, sort }, { dataSources }) => {
      return dataSources.userAPI.getUsers({ first, after, filter, sort });
    },
  },

  Mutation: {
    signUp: async (_root, { input }, { dataSources }) => {
      return dataSources.userAPI.signUp(input);
    },

    signIn: async (_root, { input }, { dataSources }) => {
      return dataSources.userAPI.signIn(input);
    },

    refreshToken: async (_root, { input }, { dataSources }) => {
      return dataSources.userAPI.refreshToken(input.token);
    },

    updateUser: async (_root, { input }, { user, dataSources }) => {
      if (!user) {
        throw new GraphQLError('Not authenticated', {
          extensions: { code: 'UNAUTHENTICATED' },
        });
      }
      return dataSources.userAPI.updateUser(user.id, input);
    },

    deleteUser: async (_root, { id }, { user, dataSources }) => {
      if (!user || user.role !== 'ADMIN') {
        throw new GraphQLError('Not authorized', {
          extensions: { code: 'FORBIDDEN' },
        });
      }
      return dataSources.userAPI.deleteUser(id);
    },

    uploadAvatar: async (_root, { file }, { user, dataSources }) => {
      if (!user) {
        throw new GraphQLError('Not authenticated', {
          extensions: { code: 'UNAUTHENTICATED' },
        });
      }
      return dataSources.userAPI.uploadAvatar(user.id, file);
    },
  },

  Subscription: {
    userUpdated: {
      subscribe: async (_root, { userId }, { pubsub }) => {
        return pubsub.subscribe(`USER_UPDATED:${userId}`);
      },
    },

    messageSent: {
      subscribe: async (_root, { chatId }, { pubsub }) => {
        return pubsub.subscribe(`MESSAGE_SENT:${chatId}`);
      },
    },
  },

  User: {
    fullName: (user) => `${user.firstName} ${user.lastName}`,
    posts: async (user, { first, after }, { dataSources }) => {
      return dataSources.postAPI.getPostsByUser(user.id, { first, after });
    },
  },
};

// Custom Scalar Types
const resolversMap = {
  Date: new Date('Invalid Date'),
  Upload: new Date('Invalid Upload'),
  JSON: new Date('Invalid JSON'),
};

// Data Source
class UserAPI {
  async getUser(id: string) {
    // Implementation
    return {};
  }

  async getUsers(options: any) {
    // Implementation with cursor-based pagination
    return {};
  }

  async signUp(input: any) {
    // Implementation
    return {};
  }

  async signIn(input: any) {
    // Implementation
    return {};
  }

  async refreshToken(token: string) {
    // Implementation
    return {};
  }

  async updateUser(id: string, input: any) {
    // Implementation
    return {};
  }

  async deleteUser(id: string) {
    // Implementation
    return true;
  }

  async uploadAvatar(userId: string, file: any) {
    // Implementation
    return '';
  }
}

// Context Builder
interface ContextValue {
  user?: any;
  dataSources: {
    userAPI: UserAPI;
  };
  pubsub: any;
  cacheControl: any;
}

const context = async ({ req }: { req: any }): Promise<ContextValue> => {
  const token = req.headers.authorization?.replace('Bearer ', '');
  let user;

  if (token) {
    user = await verifyToken(token);
  }

  return {
    user,
    dataSources: {
      userAPI: new UserAPI(),
    },
    pubsub,
    cacheControl,
  };
};

// Apollo Server Setup
const app = express();
const httpServer = http.createServer(app);

const schema = makeExecutableSchema({
  typeDefs,
  resolvers,
});

const server = new ApolloServer({
  schema,
  plugins: [
    ApolloServerPluginDrainHttpServer({ httpServer }),
  ],
  formatError: (formattedError, error) => {
    return {
      ...formattedError,
      message: formattedError.message,
      extensions: {
        ...formattedError.extensions,
        code: formattedError.extensions?.code || 'INTERNAL_SERVER_ERROR',
      },
    };
  },
});

// WebSocket Server
const wsServer = new WebSocketServer({
  server: httpServer,
  path: '/graphql',
});

useServer({ schema }, wsServer);

// Start Server
await server.start();
app.use(
  '/graphql',
  cors<cors.CorsRequest>(),
  express.json(),
  expressMiddleware(server, { context })
);

3. gRPC Service Implementation

Production gRPC Service

// user_service.proto - Protocol Buffers Definition
syntax = "proto3";

package user.v1;

option go_package = "github.com/myapp/api/gen/go/user/v1;userv1";
option java_multiple_files = true;
option java_package = "com.myapp.api.user.v1";
option java_outer_classname = "UserServiceProto";

import "google/protobuf/timestamp.proto";
import "google/protobuf/empty.proto";
import "validate/validate.proto";

// User Service
service UserService {
  // Get user by ID
  rpc GetUser(GetUserRequest) returns (GetUserResponse);

  // List users with pagination
  rpc ListUsers(ListUsersRequest) returns (ListUsersResponse);

  // Create new user
  rpc CreateUser(CreateUserRequest) returns (CreateUserResponse);

  // Update user
  rpc UpdateUser(UpdateUserRequest) returns (UpdateUserResponse);

  // Delete user
  rpc DeleteUser(DeleteUserRequest) returns (google.protobuf.Empty);

  // Batch get users
  rpc BatchGetUsers(BatchGetUsersRequest) returns (BatchGetUsersResponse);

  // User streaming
  rpc StreamUsers(StreamUsersRequest) returns (stream User);
}

// Messages
message User {
  string id = 1 [(validate.rules).string.uuid = true];
  string email = 2 [(validate.rules).string.email = true];
  string first_name = 3 [(validate.rules).string = {min_len: 1, max_len: 100}];
  string last_name = 4 [(validate.rules).string = {min_len: 1, max_len: 100}];
  string avatar_url = 5;
  Role role = 6 [(validate.rules).enum.defined_only = true];
  google.protobuf.Timestamp created_at = 7;
  google.protobuf.Timestamp updated_at = 8;
}

message GetUserRequest {
  string id = 1 [(validate.rules).string.uuid = true];
}

message GetUserResponse {
  User user = 1;
}

message ListUsersRequest {
  int32 page_size = 2 [(validate.rules).int32 = {greater_than: 0, less_than: 100}];
  string page_token = 3;
  string filter = 4;
  string sort_by = 5;
  bool sort_ascending = 6;
}

message ListUsersResponse {
  repeated User users = 1;
  string next_page_token = 2;
  int32 total_count = 3;
}

message CreateUserRequest {
  string email = 1 [(validate.rules).string.email = true];
  string password = 2 [(validate.rules).string.min_len = 8];
  string first_name = 3 [(validate.rules).string = {min_len: 1, max_len: 100}];
  string last_name = 4 [(validate.rules).string = {min_len: 1, max_len: 100}];
  Role role = 5 [(validate.rules).enum.defined_only = true];
}

message CreateUserResponse {
  User user = 1;
  string access_token = 2;
  string refresh_token = 3;
}

message UpdateUserRequest {
  string id = 1 [(validate.rules).string.uuid = true];
  string email = 2 [(validate.rules).string.email = true];
  string first_name = 3 [(validate.rules).string = {min_len: 1, max_len: 100}];
  string last_name = 4 [(validate.rules).string = {min_len: 1, max_len: 100}];
  string avatar_url = 5;
}

message UpdateUserResponse {
  User user = 1;
}

message DeleteUserRequest {
  string id = 1 [(validate.rules).string.uuid = true];
}

message BatchGetUsersRequest {
  repeated string ids = 1 [(validate.rules).repeated = {min_items: 1, max_items: 100}];
}

message BatchGetUsersResponse {
  map<string, User> users = 1;
}

message StreamUsersRequest {
  string filter = 1;
}

enum Role {
  ROLE_UNSPECIFIED = 0;
  ROLE_USER = 1;
  ROLE_ADMIN = 2;
  ROLE_MODERATOR = 3;
}

// server.go - gRPC Server Implementation in Go
package main

import (
	"context"
	"crypto/tls"
	"log"
	"net"
	"os"
	"time"

	"github.com/go-redis/redis/v8"
	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/types/known/timestamppb"

	userv1 "github.com/myapp/api/gen/go/user/v1"
)

type server struct {
	userv1.UnimplementedUserServiceServer
	redis *redis.Client
	db    Database
}

func main() {
	// Initialize dependencies
	s := &server{
		redis: redis.NewClient(&redis.Options{
			Addr:     os.Getenv("REDIS_ADDR"),
			Password: os.Getenv("REDIS_PASSWORD"),
			DB:       0,
		}),
		db: NewDatabase(),
	}

	// Create gRPC server with keepalive
	grpcServer := grpc.NewServer(
		grpc.KeepaliveParams(keepalive.ServerParameters{
			MaxConnectionIdle: 5 * time.Minute,
			Time:              10 * time.Second,
			Timeout:           1 * time.Second,
		}),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
			MinTime:             5 * time.Second,
			PermitWithoutStream: true,
		}),
		grpc.MaxRecvMsgSize(1024*1024*10), // 10MB
		grpc.MaxSendMsgSize(1024*1024*10), // 10MB
	)

	// Register service
	userv1.RegisterUserServiceServer(grpcServer, s)

	// Start server
	listener, err := net.Listen("tcp", ":50051")
	if err != nil {
		log.Fatalf("Failed to listen: %v", err)
	}

	log.Printf("gRPC server listening on :50051")
	if err := grpcServer.Serve(listener); err != nil {
		log.Fatalf("Failed to serve: %v", err)
	}
}

func (s *server) GetUser(ctx context.Context, req *userv1.GetUserRequest) (*userv1.GetUserResponse, error) {
	// Validate request
	if err := req.Validate(); err != nil {
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}

	// Try cache first
	user, err := s.getUserFromCache(ctx, req.Id)
	if err == redis.Nil {
		// Cache miss, get from database
		user, err = s.db.GetUser(ctx, req.Id)
		if err != nil {
			return nil, status.Error(codes.NotFound, "User not found")
		}

		// Set cache
		s.setUserCache(ctx, user)
	} else if err != nil {
		return nil, status.Error(codes.Internal, "Failed to get user")
	}

	return &userv1.GetUserResponse{User: user}, nil
}

func (s *server) ListUsers(ctx context.Context, req *userv1.ListUsersRequest) (*userv1.ListUsersResponse, error) {
	// Validate request
	if err := req.Validate(); err != nil {
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}

	// Default page size
	if req.PageSize == 0 {
		req.PageSize = 20
	}

	// Get users from database
	users, nextToken, total, err := s.db.ListUsers(ctx, req)
	if err != nil {
		return nil, status.Error(codes.Internal, "Failed to list users")
	}

	return &userv1.ListUsersResponse{
		Users:         users,
		NextPageToken: nextToken,
		TotalCount:    int32(total),
	}, nil
}

func (s *server) CreateUser(ctx context.Context, req *userv1.CreateUserRequest) (*userv1.CreateUserResponse, error) {
	// Validate request
	if err := req.Validate(); err != nil {
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}

	// Check if user exists
	exists, err := s.db.UserExists(ctx, req.Email)
	if err != nil {
		return nil, status.Error(codes.Internal, "Failed to check user existence")
	}
	if exists {
		return nil, status.Error(codes.AlreadyExists, "User already exists")
	}

	// Hash password
	hashedPassword, err := hashPassword(req.Password)
	if err != nil {
		return nil, status.Error(codes.Internal, "Failed to hash password")
	}

	// Create user
	user := &userv1.User{
		Id:        generateUUID(),
		Email:     req.Email,
		FirstName: req.FirstName,
		LastName:  req.LastName,
		Role:      req.Role,
		CreatedAt: timestamppb.Now(),
		UpdatedAt: timestamppb.Now(),
	}

	if err := s.db.CreateUser(ctx, user, hashedPassword); err != nil {
		return nil, status.Error(codes.Internal, "Failed to create user")
	}

	// Generate tokens
	accessToken, err := generateAccessToken(user)
	if err != nil {
		return nil, status.Error(codes.Internal, "Failed to generate access token")
	}

	refreshToken, err := generateRefreshToken(user)
	if err != nil {
		return nil, status.Error(codes.Internal, "Failed to generate refresh token")
	}

	// Invalidate cache
	s.invalidateUserCache(ctx, user.Id)

	return &userv1.CreateUserResponse{
		User:         user,
		AccessToken:  accessToken,
		RefreshToken: refreshToken,
	}, nil
}

func (s *server) UpdateUser(ctx context.Context, req *userv1.UpdateUserRequest) (*userv1.UpdateUserResponse, error) {
	// Validate request
	if err := req.Validate(); err != nil {
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}

	// Get existing user
	user, err := s.db.GetUser(ctx, req.Id)
	if err != nil {
		return nil, status.Error(codes.NotFound, "User not found")
	}

	// Update fields
	if req.Email != "" {
		user.Email = req.Email
	}
	if req.FirstName != "" {
		user.FirstName = req.FirstName
	}
	if req.LastName != "" {
		user.LastName = req.LastName
	}
	if req.AvatarUrl != "" {
		user.AvatarUrl = req.AvatarUrl
	}
	user.UpdatedAt = timestamppb.Now()

	// Save to database
	if err := s.db.UpdateUser(ctx, user); err != nil {
		return nil, status.Error(codes.Internal, "Failed to update user")
	}

	// Invalidate cache
	s.invalidateUserCache(ctx, user.Id)

	return &userv1.UpdateUserResponse{User: user}, nil
}

func (s *server) DeleteUser(ctx context.Context, req *userv1.DeleteUserRequest) (*empty.Empty, error) {
	if err := req.Validate(); err != nil {
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}

	if err := s.db.DeleteUser(ctx, req.Id); err != nil {
		return nil, status.Error(codes.Internal, "Failed to delete user")
	}

	// Invalidate cache
	s.invalidateUserCache(ctx, req.Id)

	return &empty.Empty{}, nil
}

func (s *server) BatchGetUsers(ctx context.Context, req *userv1.BatchGetUsersRequest) (*userv1.BatchGetUsersResponse, error) {
	if err := req.Validate(); err != nil {
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}

	users := make(map[string]*userv1.User)
	for _, id := range req.Ids {
		user, err := s.db.GetUser(ctx, id)
		if err != nil {
			continue // Skip missing users
		}
		users[id] = user
	}

	return &userv1.BatchGetUsersResponse{Users: users}, nil
}

func (s *server) StreamUsers(req *userv1.StreamUsersRequest, stream userv1.UserService_StreamUsersServer) error {
	// Stream users from database
	err := s.db.StreamUsers(stream.Context(), req.Filter, func(user *userv1.User) error {
		return stream.Send(user)
	})

	if err != nil {
		return status.Error(codes.Internal, "Failed to stream users")
	}

	return nil
}

4. API Versioning

Versioning Strategies

// API Versioning Implementation
// Multiple versioning strategies

import express from 'express';

// Strategy 1: URL Path Versioning
// /api/v1/users, /api/v2/users
const v1Router = express.Router();
const v2Router = express.Router();

v1Router.get('/users', (req, res) => {
  // V1 implementation
  res.json({
    users: [
      {
        id: '1',
        email: 'user@example.com',
        first_name: 'John',
        last_name: 'Doe',
      },
    ],
  });
});

v2Router.get('/users', (req, res) => {
  // V2 implementation with different response format
  res.json({
    data: [
      {
        id: '1',
        email: 'user@example.com',
        firstName: 'John',
        lastName: 'Doe',
        avatar: null,
      },
    ],
    meta: {
      page: 1,
      limit: 20,
      total: 100,
    },
  });
});

app.use('/api/v1', v1Router);
app.use('/api/v2', v2Router);

// Strategy 2: Header Versioning
// Accept: application/vnd.myapi.v1+json
const headerVersionRouter = express.Router();

headerVersionRouter.get('/users', (req, res) => {
  const acceptHeader = req.headers.accept || '';
  const version = acceptHeader.match(/application\.vnd\.myapi\.v(\d)\+json/)?.[1] || '1';

  switch (version) {
    case '1':
      res.json({ v1: true, users: [] });
      break;
    case '2':
      res.json({ v2: true, data: [], meta: {} });
      break;
    default:
      res.status(400).json({ error: 'Unsupported API version' });
  }
});

// Strategy 3: Query Parameter Versioning
// /api/users?version=2
const queryVersionRouter = express.Router();

queryVersionRouter.get('/users', (req, res) => {
  const version = req.query.version || '1';

  switch (version) {
    case '1':
      res.json({ v1: true, users: [] });
      break;
    case '2':
      res.json({ v2: true, data: [], meta: {} });
      break;
    default:
      res.status(400).json({ error: 'Unsupported API version' });
  }
});

// Version deprecation strategy
class APIVersionManager {
  private versions: Map<string, Date> = new Map();

  constructor() {
    // Register versions and their sunset dates
    this.versions.set('v1', new Date('2024-12-31'));
    this.versions.set('v2', new Date('2025-12-31'));
  }

  isDeprecated(version: string): boolean {
    const sunsetDate = this.versions.get(version);
    return sunsetDate ? new Date() > sunsetDate : false;
  }

  getSunsetWarning(version: string): string | null {
    const sunsetDate = this.versions.get(version);
    if (!sunsetDate) return null;

    const daysUntilSunset = Math.ceil(
      (sunsetDate.getTime() - Date.now()) / (1000 * 60 * 60 * 24)
    );

    if (daysUntilSunset > 0) {
      return `API ${version} will be deprecated on ${sunsetDate.toISOString()} (${daysUntilSunset} days remaining)`;
    }

    return `API ${version} is deprecated and will be removed soon`;
  }
}

// Version-aware middleware
const versionManager = new APIVersionManager();

app.use((req, res, next) => {
  const version = req.path.match(/\/api\/(v\d+)/)?.[1] || 'v1';

  if (versionManager.isDeprecated(version)) {
    res.setHeader('X-API-Deprecation', versionManager.getSunsetWarning(version)!);
    res.setHeader('Sunset', versionManager.versions.get(version)!.toUTCString());
  }

  next();
});

5. Real-time APIs

WebSocket Implementation

// WebSocket API Implementation
// Complete WebSocket server with authentication and rooms

import { WebSocketServer, WebSocket } from 'ws';
import { createServer } from 'http';
import jwt from 'jsonwebtoken';
import Redis from 'ioredis';

interface WebSocketMessage {
  type: string;
  payload: any;
  roomId?: string;
  userId?: string;
}

interface AuthenticatedWebSocket extends WebSocket {
  userId?: string;
  rooms?: Set<string>;
  isAlive?: boolean;
}

class WebSocketServer {
  private wss: WebSocketServer;
  private redis: Redis;
  private clients: Map<WebSocket, AuthenticatedWebSocket> = new Map();
  private rooms: Map<string, Set<WebSocket>> = new Map();

  constructor(httpServer: any) {
    this.wss = new WebSocket.WebSocketServer({ server: httpServer });
    this.redis = new Redis(process.env.REDIS_URL);

    this.wss.on('connection', this.handleConnection.bind(this));
    this.startHeartbeat();
  }

  private async handleConnection(ws: AuthenticatedWebSocket, req: any) {
    const token = new URL(req.url, 'http://localhost').searchParams.get('token');

    if (!token) {
      ws.close(1008, 'Authentication required');
      return;
    }

    try {
      // Verify JWT token
      const decoded = jwt.verify(token, process.env.JWT_SECRET!) as any;
      ws.userId = decoded.id;
      ws.rooms = new Set();
      ws.isAlive = true;

      this.clients.set(ws, ws);

      ws.on('message', (data: string) => {
        this.handleMessage(ws, JSON.parse(data));
      });

      ws.on('close', () => {
        this.handleDisconnection(ws);
      });

      ws.on('pong', () => {
        ws.isAlive = true;
      });

      // Send welcome message
      this.sendToClient(ws, {
        type: 'connected',
        payload: {
          userId: ws.userId,
          timestamp: new Date().toISOString(),
        },
      });
    } catch (error) {
      ws.close(1008, 'Invalid token');
    }
  }

  private handleMessage(ws: AuthenticatedWebSocket, message: WebSocketMessage) {
    switch (message.type) {
      case 'join_room':
        this.joinRoom(ws, message.roomId!);
        break;

      case 'leave_room':
        this.leaveRoom(ws, message.roomId!);
        break;

      case 'broadcast':
        this.broadcastToRoom(ws, message.roomId!, message.payload);
        break;

      case 'direct_message':
        this.sendDirectMessage(ws, message.userId!, message.payload);
        break;

      default:
        this.sendError(ws, 'Unknown message type');
    }
  }

  private joinRoom(ws: AuthenticatedWebSocket, roomId: string) {
    if (!ws.rooms!.has(roomId)) {
      ws.rooms!.add(roomId);

      if (!this.rooms.has(roomId)) {
        this.rooms.set(roomId, new Set());
      }

      this.rooms.get(roomId)!.add(ws);

      this.sendToClient(ws, {
        type: 'room_joined',
        payload: { roomId },
      });

      // Notify others in room
      this.broadcastToRoom(ws, roomId, {
        type: 'user_joined',
        payload: { userId: ws.userId },
      });
    }
  }

  private leaveRoom(ws: AuthenticatedWebSocket, roomId: string) {
    if (ws.rooms!.has(roomId)) {
      ws.rooms!.delete(roomId);
      this.rooms.get(roomId)!.delete(ws);

      this.sendToClient(ws, {
        type: 'room_left',
        payload: { roomId },
      });

      // Notify others in room
      this.broadcastToRoom(ws, roomId, {
        type: 'user_left',
        payload: { userId: ws.userId },
      });
    }
  }

  private broadcastToRoom(ws: AuthenticatedWebSocket, roomId: string, payload: any) {
    const room = this.rooms.get(roomId);
    if (!room) return;

    const message = {
      type: 'broadcast',
      payload: {
        roomId,
        userId: ws.userId,
        data: payload,
        timestamp: new Date().toISOString(),
      },
    };

    room.forEach((client) => {
      if (client !== ws && client.readyState === WebSocket.OPEN) {
        client.send(JSON.stringify(message));
      }
    });
  }

  private sendDirectMessage(ws: AuthenticatedWebSocket, targetUserId: string, payload: any) {
    const targetClient = Array.from(this.clients.values()).find(
      (client) => client.userId === targetUserId
    );

    if (targetClient && targetClient.readyState === WebSocket.OPEN) {
      this.sendToClient(targetClient, {
        type: 'direct_message',
        payload: {
          from: ws.userId,
          data: payload,
          timestamp: new Date().toISOString(),
        },
      });
    }
  }

  private sendToClient(ws: AuthenticatedWebSocket, message: any) {
    if (ws.readyState === WebSocket.OPEN) {
      ws.send(JSON.stringify(message));
    }
  }

  private sendError(ws: AuthenticatedWebSocket, error: string) {
    this.sendToClient(ws, {
      type: 'error',
      payload: { error },
    });
  }

  private handleDisconnection(ws: AuthenticatedWebSocket) {
    // Leave all rooms
    ws.rooms!.forEach((roomId) => {
      this.rooms.get(roomId)?.delete(ws);
    });

    this.clients.delete(ws);
  }

  private startHeartbeat() {
    const interval = setInterval(() => {
      this.wss.clients.forEach((ws: any) => {
        if (ws.isAlive === false) {
          return ws.terminate();
        }

        ws.isAlive = false;
        ws.ping();
      });
    }, 30000);

    this.wss.on('close', () => {
      clearInterval(interval);
    });
  }
}

// SSE (Server-Sent Events) Implementation
class SSEServer {
  private clients: Map<string, Response> = new Map();

  async handleSSE(req: Request, res: Response) {
    const userId = req.user!.id;

    // Set SSE headers
    res.setHeader('Content-Type', 'text/event-stream');
    res.setHeader('Cache-Control', 'no-cache');
    res.setHeader('Connection', 'keep-alive');
    res.setHeader('X-Accel-Buffering', 'no');

    this.clients.set(userId, res);

    // Send initial connection message
    this.sendEvent(userId, 'connected', { timestamp: Date.now() });

    // Handle client disconnect
    req.on('close', () => {
      this.clients.delete(userId);
    });
  }

  sendEvent(userId: string, event: string, data: any) {
    const client = this.clients.get(userId);
    if (!client) return;

    client.write(`event: ${event}\n`);
    client.write(`data: ${JSON.stringify(data)}\n\n`);
  }

  broadcast(event: string, data: any) {
    this.clients.forEach((client) => {
      this.sendEvent(
        Array.from(this.clients.keys()).find((id) => this.clients.get(id) === client)!,
        event,
        data
      );
    });
  }
}

// Webhook Implementation
class WebhookService {
  async sendWebhook(url: string, payload: any, retries = 3) {
    for (let i = 0; i < retries; i++) {
      try {
        const response = await fetch(url, {
          method: 'POST',
          headers: {
            'Content-Type': 'application/json',
            'User-Agent': 'MyApp/1.0',
          },
          body: JSON.stringify(payload),
          signal: AbortSignal.timeout(5000), // 5 second timeout
        });

        if (response.ok) {
          return { success: true, statusCode: response.status };
        }

        // Retry on 5xx errors
        if (response.status >= 500 && i < retries - 1) {
          await this.exponentialBackoff(i);
          continue;
        }

        return {
          success: false,
          statusCode: response.status,
          error: 'Webhook delivery failed',
        };
      } catch (error) {
        if (i < retries - 1) {
          await this.exponentialBackoff(i);
          continue;
        }

        return {
          success: false,
          error: 'Webhook delivery failed',
        };
      }
    }
  }

  private async exponentialBackoff(attempt: number) {
    const delay = Math.min(1000 * Math.pow(2, attempt), 10000);
    await new Promise((resolve) => setTimeout(resolve, delay));
  }
}

6. Decision Trees

API Type Selection

Communication requirements?
│
├─ Simple CRUD → REST
├─ Complex queries → GraphQL
├─ High performance / Microservices → gRPC
├─ Real-time bidirectional → WebSocket
├─ Server-to-client streaming → SSE
├─ Server-to-server notifications → Webhooks
└─ File upload/download → REST with multipart

Authentication Strategy

Client type and security requirements?
│
├─ Server-to-server → API Keys / JWT
├─ Web application → OAuth 2.0 / PKCE
├─ Mobile app → OAuth 2.0 / Refresh tokens
├─ IoT devices → JWT / X.509 certificates
├─ Internal microservices → mTLS / JWT
└─ Third-party integration → OAuth 2.0

7. Anti-Patterns to Avoid

1. N+1 queries: Always optimize data fetching
2. Missing versioning: Always version your APIs
3. Inconsistent error handling: Use standard error formats
4. No rate limiting: Always implement rate limits
5. Ignoring security: Always authenticate and authorize
6. Poor pagination: Use cursor-based for large datasets
7. No documentation: Always document your APIs
8. Tight coupling: Design for loose coupling
9. Missing validation: Always validate input
10. No testing: Always test your APIs

8. Quality Checklist

Before considering API production-ready:

• API documentation complete (OpenAPI/Swagger)
• Authentication and authorization implemented
• Input validation on all endpoints
• Error handling standardized
• Rate limiting configured
• Logging and monitoring enabled
• Caching strategy implemented
• Pagination implemented
• API versioning strategy defined
• Security headers configured
• CORS properly configured
• Request/response validation
• Unit tests passing
• Integration tests passing
• Load testing performed
• API gateway configured
• Backup and disaster recovery planned
• SLA requirements met
• SDK documentation provided
• Changelog maintained

This comprehensive skill definition provides complete guidance for API engineering across modern architectures.