Python下载实战：高效技巧与进阶指南

Python下载实战技巧技术文章大纲

文件下载基础

• 使用urllib.request模块进行简单下载
• 使用requests库实现高效下载
• 处理HTTP状态码和异常情况

大文件分块下载

• 使用流式传输避免内存溢出
• 实现进度条显示下载进度
• 断点续传的实现方法

多线程/异步下载

• 使用threading模块实现多线程下载
• 使用asyncio实现异步下载
• 控制并发数避免服务器拒绝

下载加速技巧

• 连接复用和会话保持
• 使用CDN加速下载
• 压缩传输减少数据量

特殊场景处理

• 处理需要认证的下载链接
• 绕过反爬虫机制
• 下载动态生成的内容

安全与验证

• 校验文件完整性(MD5/SHA)
• 处理SSL证书验证
• 防范恶意文件下载

实用工具与库

• wget库的替代实现
• aria2c的Python封装
• 自制下载管理器框架

性能优化

• 缓冲区大小调整
• DNS缓存优化
• TCP参数调优

错误处理与日志

• 常见下载错误分类处理
• 设计重试机制
• 记录详细下载日志

实战案例

• 图片批量下载器实现
• 视频分段下载合并
• 云存储文件同步工具

• cmake_minimum_required(VERSION 3.5)
  project(unitree_sdk2 VERSION 2.0.0)

  ## Project Options
  option(BUILD_EXAMPLES "Build examples" ON)

  ## Set compiler to use c++ 17 features
  set(CMAKE_CXX_STANDARD 17)
  set(CMAKE_CXX_EXTENSIONS OFF)
  set(CMAKE_CXX_STANDARD_REQUIRED ON)

  ## Chosse build type
  set(default_build_type "Release")
  if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
      message(STATUS "Setting build type to '${default_build_type}' as none was specified.")
      set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
              STRING "Choose the type of build." FORCE)
      # Set the possible values of build type for cmake-gui
      set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
              "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
  endif ()

  ## Use GNUInstallDirs to install libraries into correct locations on all platforms.
  include(GNUInstallDirs)

  ## Put all binary files into /bin and libraries into /lib
  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR})

  ## Check system architecture
  message(STATUS "Current system architecture: ${CMAKE_SYSTEM_PROCESSOR}")

  ## 添加 OpenCV 支持
  # 首先尝试使用 find_package 查找 OpenCV
  find_package(OpenCV QUIET)

  if(NOT OpenCV_FOUND)
      # 如果标准方法找不到，尝试手动设置路径
      message(STATUS "OpenCV not found with standard method, trying manual configuration...")
      
      # 设置可能的 OpenCV 路径
      set(OPENCV_POSSIBLE_INCLUDE_PATHS
          "/usr/local/include/opencv4"
          "/usr/include/opencv4"
          "/usr/local/include"
          "/usr/include"
      )
      
      set(OPENCV_POSSIBLE_LIB_PATHS
          "/usr/local/lib/aarch64-linux-gnu"
          "/usr/local/lib"
          "/usr/lib/aarch64-linux-gnu"
          "/usr/lib"
      )
      
      # 查找头文件
      find_path(OPENCV_INCLUDE_DIRS
          NAMES opencv2/opencv.hpp
          PATHS ${OPENCV_POSSIBLE_INCLUDE_PATHS}
          NO_DEFAULT_PATH
      )
      
      # 查找库文件
      find_library(OPENCV_CORE_LIB
          NAMES opencv_core
          PATHS ${OPENCV_POSSIBLE_LIB_PATHS}
          NO_DEFAULT_PATH
      )
      
      find_library(OPENCV_HIGHGUI_LIB
          NAMES opencv_highgui
          PATHS ${OPENCV_POSSIBLE_LIB_PATHS}
          NO_DEFAULT_PATH
      )
      
      find_library(OPENCV_VIDEOIO_LIB
          NAMES opencv_videoio
          PATHS ${OPENCV_POSSIBLE_LIB_PATHS}
          NO_DEFAULT_PATH
      )
      
      find_library(OPENCV_IMGPROC_LIB
          NAMES opencv_imgproc
          PATHS ${OPENCV_POSSIBLE_LIB_PATHS}
          NO_DEFAULT_PATH
      )
      
      if(OPENCV_INCLUDE_DIRS AND OPENCV_CORE_LIB AND OPENCV_HIGHGUI_LIB AND OPENCV_VIDEOIO_LIB)
          set(OpenCV_FOUND TRUE)
          set(OpenCV_INCLUDE_DIRS ${OPENCV_INCLUDE_DIRS})
          set(OpenCV_LIBS ${OPENCV_CORE_LIB} ${OPENCV_HIGHGUI_LIB} ${OPENCV_VIDEOIO_LIB} ${OPENCV_IMGPROC_LIB})
          message(STATUS "OpenCV found manually:")
          message(STATUS "    Include dirs: ${OpenCV_INCLUDE_DIRS}")
          message(STATUS "    Libraries: ${OpenCV_LIBS}")
      else()
          message(WARNING "OpenCV not found. Examples requiring OpenCV will not be built.")
          set(OpenCV_FOUND FALSE)
      endif()
  else()
      message(STATUS "OpenCV found: ${OpenCV_VERSION}")
      message(STATUS "OpenCV libraries: ${OpenCV_LIBS}")
      message(STATUS "OpenCV include dirs: ${OpenCV_INCLUDE_DIRS}")
  endif()

  ## Import thirdparty libraries
  add_subdirectory(thirdparty)

  ## Import Unitree SDK2 library
  set(UNITREE_SDK_PATH ${CMAKE_CURRENT_LIST_DIR}/lib/${CMAKE_SYSTEM_PROCESSOR})
  find_library(UNITREE_SDK_LIB unitree_sdk2 PATHS ${UNITREE_SDK_PATH} NO_DEFAULT_PATH)

  if (NOT UNITREE_SDK_LIB)
      message(FATAL_ERROR "Unitree SDK library for the architecture is not found")
  else ()
      message(STATUS "Unitree SDK library found at: ${UNITREE_SDK_LIB}")
  endif ()

  message(STATUS "Importing: ${UNITREE_SDK_LIB}")

  find_package(Threads REQUIRED)

  add_library(unitree_sdk2 STATIC IMPORTED GLOBAL)
  set_target_properties(unitree_sdk2 PROPERTIES
          IMPORTED_LOCATION ${UNITREE_SDK_LIB})
  target_link_libraries(unitree_sdk2 INTERFACE ddsc ddscxx Threads::Threads)
  target_include_directories(unitree_sdk2 INTERFACE
          $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
          $<INSTALL_INTERFACE:include>)

  # 如果找到了 OpenCV，将其添加到 unitree_sdk2 的接口中
  if(OpenCV_FOUND)
      target_include_directories(unitree_sdk2 INTERFACE ${OpenCV_INCLUDE_DIRS})
      # 注意：这里我们不直接链接 OpenCV 库，因为不是所有目标都需要
      # 具体的链接将在各个示例的 CMakeLists.txt 中处理
  endif()

  if (BUILD_EXAMPLES)
      add_subdirectory(example)
  endif ()

  ## Install the library
  install(DIRECTORY include/
      DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

  install(FILES ${UNITREE_SDK_LIB}
      DESTINATION ${CMAKE_INSTALL_LIBDIR})

  install(FILES cmake/unitree_sdk2Targets.cmake
      DESTINATION lib/cmake/unitree_sdk2)

  include(CMakePackageConfigHelpers)
  write_basic_package_version_file(
      unitree_sdk2ConfigVersion.cmake
      VERSION "${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH}"
      COMPATIBILITY ExactVersion)

  configure_file(cmake/unitree_sdk2Config.cmake.in unitree_sdk2Config.cmake @ONLY)
  install(FILES "${CMAKE_CURRENT_BINARY_DIR}/unitree_sdk2Config.cmake"
      "${CMAKE_CURRENT_BINARY_DIR}/unitree_sdk2ConfigVersion.cmake"
      DESTINATION lib/cmake/unitree_sdk2)